diff --git a/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go b/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go index d734803..4631978 100644 --- a/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go +++ b/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go @@ -11,6 +11,7 @@ import ( "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/config" "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/helm" "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/operate" + "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/register" "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/users" "github.com/spf13/cobra" @@ -53,6 +54,7 @@ func NewCmd(streams genericclioptions.IOStreams) *cobra.Command { // cmd.AddCommand(migrate.NewInstallCmd(streams)) cmd.AddCommand(config.NewCmd(streams)) cmd.AddCommand(helm.NewHelmCmd(streams)) + register.SetupRegisterClusterCmd(cmd, streams) // cmd.Flags().BoolVar(&o.listNamespaces, "list", o.listNamespaces, "if true, print the list of all namespaces in the current KUBECONFIG") o.configFlags.AddFlags(cmd.Flags()) diff --git a/cmd/kubectl-k8ssandra/register/command.go b/cmd/kubectl-k8ssandra/register/command.go new file mode 100644 index 0000000..95de8a5 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/command.go @@ -0,0 +1,80 @@ +package register + +import ( + "fmt" + + "github.com/charmbracelet/log" + "github.com/k8ssandra/k8ssandra-client/pkg/registration" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +var RegisterClusterCmd = &cobra.Command{ + Use: "register [flags]", + Short: "register a data plane into the control plane.", + Long: `register creates a ServiceAccount on a source cluster, copies its credentials and then creates a secret containing them on the destination cluster. It then also creates a ClientConfig on the destination cluster to reference the secret.`, + Run: entrypoint, +} + +func SetupRegisterClusterCmd(cmd *cobra.Command, streams genericclioptions.IOStreams) { + RegisterClusterCmd.Flags().String("source-kubeconfig", + "", + "path to source cluster's kubeconfig file - defaults to KUBECONFIG then ~/.kube/config") + RegisterClusterCmd.Flags().String("dest-kubeconfig", + "", + "path to destination cluster's kubeconfig file - defaults to KUBECONFIG then ~/.kube/config") + RegisterClusterCmd.Flags().String("source-context", "", "context name for source cluster") + RegisterClusterCmd.Flags().String("dest-context", "", "context name for destination cluster") + RegisterClusterCmd.Flags().String("source-namespace", "k8ssandra-operator", "namespace containing service account for source cluster") + RegisterClusterCmd.Flags().String("dest-namespace", "k8ssandra-operator", "namespace where secret and clientConfig will be created on destination cluster") + RegisterClusterCmd.Flags().String("serviceaccount-name", "k8ssandra-operator", "serviceaccount name for destination cluster") + RegisterClusterCmd.Flags().String("destination-name", "", "name for remote clientConfig and secret on destination cluster") + + if err := RegisterClusterCmd.MarkFlagRequired("source-context"); err != nil { + panic(err) + + } + if err := RegisterClusterCmd.MarkFlagRequired("dest-context"); err != nil { + panic(err) + } + cmd.AddCommand(RegisterClusterCmd) +} + +func entrypoint(cmd *cobra.Command, args []string) { + executor := NewRegistrationExecutorFromRegisterClusterCmd(*cmd) + + for i := 0; i < 30; i++ { + res := executor.RegisterCluster() + switch v := res.(type) { + case RetryableError: + log.Info("Registration continuing", "msg", v.Error()) + continue + case nil: + log.Info("Registration completed successfully") + return + case NonRecoverableError: + panic(fmt.Sprintf("Registration failed: %s", v.Error())) + } + } + fmt.Println("Registration failed - retries exceeded") +} + +func NewRegistrationExecutorFromRegisterClusterCmd(cmd cobra.Command) *RegistrationExecutor { + + destName := cmd.Flag("destination-name").Value.String() + srcContext := cmd.Flag("source-context").Value.String() + if destName == "" { + destName = registration.CleanupForKubernetes(srcContext) + } + return &RegistrationExecutor{ + SourceKubeconfig: cmd.Flag("source-kubeconfig").Value.String(), + DestKubeconfig: cmd.Flag("dest-kubeconfig").Value.String(), + SourceContext: srcContext, + DestContext: cmd.Flag("dest-context").Value.String(), + SourceNamespace: cmd.Flag("source-namespace").Value.String(), + DestNamespace: cmd.Flag("dest-namespace").Value.String(), + ServiceAccount: cmd.Flag("serviceaccount-name").Value.String(), + Context: cmd.Context(), + DestinationName: destName, + } +} diff --git a/cmd/kubectl-k8ssandra/register/errors.go b/cmd/kubectl-k8ssandra/register/errors.go new file mode 100644 index 0000000..1a86f87 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/errors.go @@ -0,0 +1,17 @@ +package register + +type RetryableError struct { + Message string +} + +func (e RetryableError) Error() string { + return e.Message +} + +type NonRecoverableError struct { + Message string +} + +func (e NonRecoverableError) Error() string { + return e.Message +} diff --git a/cmd/kubectl-k8ssandra/register/register_test.go b/cmd/kubectl-k8ssandra/register/register_test.go new file mode 100644 index 0000000..56f1cd8 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/register_test.go @@ -0,0 +1,159 @@ +package register + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + configapi "github.com/k8ssandra/k8ssandra-operator/apis/config/v1beta1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestRegister(t *testing.T) { + require := require.New(t) + client1 := (*multiEnv)[0].GetClientInNamespace("source-namespace") + client2 := (*multiEnv)[1].GetClientInNamespace("dest-namespace") + require.NoError(client1.Create((*multiEnv)[0].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "source-namespace"}})) + require.NoError(client2.Create((*multiEnv)[1].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dest-namespace"}})) + + testDir, err := os.MkdirTemp("", "k8ssandra-operator-test-****") + require.NoError(err) + // buildDir := filepath.Join(envtest.RootDir(), "build") + // testDir := filepath.Join(buildDir, time.Now()) + + // if _, err := os.Stat(testDir); os.IsNotExist(err) { + // err := os.MkdirAll(testDir, os.ModePerm) + // require.NoError(err) + // } else if err != nil { + // require.NoError(err) + // } + t.Cleanup(func() { + require.NoError(os.RemoveAll(testDir)) + }) + + kc1, err := (*multiEnv)[0].GetKubeconfig() + require.NoError(err) + f1, err := os.Create(testDir + "/kubeconfig1") + require.NoError(err) + t.Cleanup(func() { + require.NoError(f1.Close()) + }) + _, err = f1.Write(kc1) + require.NoError(err) + + f2, err := os.Create(testDir + "/kubeconfig2") + require.NoError(err) + t.Cleanup(func() { + require.NoError(f2.Close()) + }) + + kc2, err := (*multiEnv)[1].GetKubeconfig() + require.NoError(err) + _, err = f2.Write(kc2) + require.NoError(err) + + ex := RegistrationExecutor{ + SourceKubeconfig: testDir + "/kubeconfig1", + DestKubeconfig: testDir + "/kubeconfig2", + SourceContext: "default-context", + DestContext: "default-context", + SourceNamespace: "source-namespace", + DestNamespace: "dest-namespace", + ServiceAccount: "k8ssandra-operator", + Context: context.TODO(), + DestinationName: "test-destination", + } + ctx := context.Background() + + require.Eventually(func() bool { + res := ex.RegisterCluster() + switch v := res.(type) { + case RetryableError: + if res.Error() == "no secret found for service account k8ssandra-operator" { + return true + } + case nil: + return true + case NonRecoverableError: + panic(fmt.Sprintf("Registration failed: %s", v.Error())) + } + return false + }, time.Second*30, time.Second*5) + + // This relies on a controller that is not running in the envtest. + + desiredSaSecret := &corev1.Secret{} + require.NoError(client1.Get(context.Background(), client.ObjectKey{Name: "k8ssandra-operator-secret", Namespace: "source-namespace"}, desiredSaSecret)) + patch := client.MergeFrom(desiredSaSecret.DeepCopy()) + desiredSaSecret.Data = map[string][]byte{ + "token": []byte("test-token"), + "ca.crt": []byte("test-ca"), + } + require.NoError(client1.Patch(ctx, desiredSaSecret, patch)) + + desiredSa := &corev1.ServiceAccount{} + require.NoError(client1.Get( + context.Background(), + client.ObjectKey{Name: "k8ssandra-operator", Namespace: "source-namespace"}, + desiredSa)) + + patch = client.MergeFrom(desiredSa.DeepCopy()) + desiredSa.Secrets = []corev1.ObjectReference{ + { + Name: "k8ssandra-operator-secret", + }, + } + require.NoError(client1.Patch(ctx, desiredSa, patch)) + + // Continue reconciliation + + require.Eventually(func() bool { + res := ex.RegisterCluster() + return res == nil + }, time.Second*300, time.Second*1) + + if err := configapi.AddToScheme(client2.Scheme()); err != nil { + require.NoError(err) + } + destSecret := &corev1.Secret{} + require.Eventually(func() bool { + err = client2.Get(ctx, + client.ObjectKey{Name: "test-destination", Namespace: "dest-namespace"}, destSecret) + if err != nil { + t.Log("didn't find dest secret") + return false + } + clientConfig := &configapi.ClientConfig{} + err = client2.Get(ctx, + client.ObjectKey{Name: "test-destination", Namespace: "dest-namespace"}, clientConfig) + if err != nil { + t.Log("didn't find dest client config") + return false + } + return err == nil + }, time.Second*60, time.Second*5) + + destKubeconfig := ClientConfigFromSecret(destSecret) + require.Equal( + desiredSaSecret.Data["ca.crt"], + destKubeconfig.Clusters["test-destination"].CertificateAuthorityData) + + require.Equal( + string(desiredSaSecret.Data["token"]), + destKubeconfig.AuthInfos["test-destination"].Token) +} + +func ClientConfigFromSecret(s *corev1.Secret) clientcmdapi.Config { + out, err := clientcmd.Load(s.Data["kubeconfig"]) + if err != nil { + panic(err) + } + return *out +} diff --git a/cmd/kubectl-k8ssandra/register/registration.go b/cmd/kubectl-k8ssandra/register/registration.go new file mode 100644 index 0000000..71f68a6 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/registration.go @@ -0,0 +1,147 @@ +package register + +import ( + "context" + "fmt" + + "github.com/charmbracelet/log" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/k8ssandra/k8ssandra-client/pkg/registration" + configapi "github.com/k8ssandra/k8ssandra-operator/apis/config/v1beta1" +) + +type RegistrationExecutor struct { + DestinationName string + SourceKubeconfig string + DestKubeconfig string + SourceContext string + DestContext string + SourceNamespace string + DestNamespace string + ServiceAccount string + Context context.Context +} + +func getDefaultSecret(saNamespace, saName string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName + "-secret", + Namespace: saNamespace, + Annotations: map[string]string{ + "kubernetes.io/service-account.name": saName, + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + } +} + +func getDefaultServiceAccount(saName, saNamespace string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: saNamespace, + }, + } +} + +func (e *RegistrationExecutor) RegisterCluster() error { + log.Printf("Registering cluster from %s Context: %s to %s Context: %s", + registration.GetKubeconfigFileLocation(e.SourceKubeconfig), e.SourceContext, + registration.GetKubeconfigFileLocation(e.DestKubeconfig), e.DestContext, + ) + if e.SourceContext == e.DestContext && e.SourceKubeconfig == e.DestKubeconfig { + return NonRecoverableError{Message: "source and destination context and kubeconfig are the same, you should not register the same cluster to itself. Reference it by leaving the k8sContext field blank instead"} + } + srcClient, err := registration.GetClient(e.SourceKubeconfig, e.SourceContext) + if err != nil { + return RetryableError{Message: err.Error()} + } + destClient, err := registration.GetClient(e.DestKubeconfig, e.DestContext) + if err != nil { + return RetryableError{Message: err.Error()} + } + // Get ServiceAccount + serviceAccount := &corev1.ServiceAccount{} + if err := srcClient.Get(e.Context, client.ObjectKey{Name: e.ServiceAccount, Namespace: e.SourceNamespace}, serviceAccount); err != nil { + if apierrors.IsNotFound(err) { + if err := srcClient.Create(e.Context, getDefaultServiceAccount(e.ServiceAccount, e.SourceNamespace)); err != nil { + return RetryableError{Message: err.Error()} + } + } + return RetryableError{Message: err.Error()} + } + // Get a secret in this namespace which holds the service account token + secretsList := &corev1.SecretList{} + if err := srcClient.List(e.Context, secretsList, client.InNamespace(e.SourceNamespace)); err != nil { + return RetryableError{Message: err.Error()} + } + var secret *corev1.Secret + for _, s := range secretsList.Items { + if s.Annotations["kubernetes.io/service-account.name"] == e.ServiceAccount && s.Type == corev1.SecretTypeServiceAccountToken { + secret = &s + break + } + } + if secret == nil { + secret = getDefaultSecret(e.SourceNamespace, e.ServiceAccount) + if err := srcClient.Create(e.Context, secret); err != nil { + return RetryableError{Message: err.Error()} + } + return RetryableError{Message: fmt.Sprintf("no secret found for service account %s", e.ServiceAccount)} + } + + // Create Secret on destination cluster + host, err := registration.KubeconfigToHost(e.SourceKubeconfig, e.SourceContext) + if err != nil { + return RetryableError{Message: err.Error()} + } + saConfig, err := registration.TokenToKubeconfig(*secret, host, e.DestinationName) + if err != nil { + return RetryableError{fmt.Sprintf("error converting token to kubeconfig: %s, secret: %#v", err.Error(), secret)} + } + secretData, err := clientcmd.Write(saConfig) + if err != nil { + return RetryableError{Message: err.Error()} + } + destSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: e.DestinationName, + Namespace: e.DestNamespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "kubeconfig": secretData, + }, + } + if err := destClient.Create(e.Context, &destSecret); err != nil && !errors.IsAlreadyExists(err) { + return RetryableError{fmt.Sprintf("error creating secret. err: %s sa %s", err, e.ServiceAccount)} + } + + // Create ClientConfig on destination cluster + if err := configapi.AddToScheme(destClient.Scheme()); err != nil { + return RetryableError{Message: err.Error()} + } + destClientConfig := configapi.ClientConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: e.DestinationName, + Namespace: e.DestNamespace, + }, + Spec: configapi.ClientConfigSpec{ + KubeConfigSecret: corev1.LocalObjectReference{ + Name: e.DestinationName, + }, + ContextName: e.DestinationName, + }, + } + if err := destClient.Create(e.Context, &destClientConfig); err != nil && !errors.IsAlreadyExists(err) { + return RetryableError{Message: err.Error()} + } + return nil +} diff --git a/cmd/kubectl-k8ssandra/register/suite_test.go b/cmd/kubectl-k8ssandra/register/suite_test.go new file mode 100644 index 0000000..0211a89 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/suite_test.go @@ -0,0 +1,19 @@ +package register + +import ( + "os" + "testing" + + "github.com/k8ssandra/k8ssandra-client/internal/envtest" +) + +var ( + multiEnv *envtest.MultiK8sEnvironment +) + +func TestMain(m *testing.M) { + // metrics.DefaultBindAddress = "0" This no longer appears to exist... + os.Exit(envtest.RunMulti(m, func(e *envtest.MultiK8sEnvironment) { + multiEnv = e + }, 2)) +} diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index e04bf5d..120c290 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -3,17 +3,18 @@ package envtest import ( "context" "path/filepath" - "runtime" "strings" "testing" cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" controlapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" + "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" - apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/kubectl/pkg/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,51 +22,67 @@ import ( ) func Run(m *testing.M, setupFunc func(e *Environment)) (code int) { - env := NewEnvironment() - env.start() + ctx := ctrl.SetupSignalHandler() + env := NewEnvironment(ctx) + env.Start() setupFunc(env) exitCode := m.Run() - env.stop() + env.Stop() return exitCode } type Environment struct { - intClient client.Client + client client.Client env *envtest.Environment cancelManager context.CancelFunc Context context.Context } -func NewEnvironment() *Environment { +func NewEnvironment(ctx context.Context) *Environment { env := &Environment{} env.env = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join(RootDir(), "testfiles", "crd")}, + CRDDirectoryPaths: []string{ + filepath.Join(RootDir(), "testfiles", "crd"), + }, ErrorIfCRDPathMissing: true, } - - ctx := ctrl.SetupSignalHandler() ctx, cancel := context.WithCancel(ctx) env.Context = ctx env.cancelManager = cancel return env } -// https://stackoverflow.com/questions/31873396/is-it-possible-to-get-the-current-root-of-package-structure-as-a-string-in-golan -func RootDir() string { - _, b, _, _ := runtime.Caller(0) - return filepath.Join(filepath.Dir(b), "../..") +func (e *Environment) GetClientInNamespace(namespace string) client.Client { + c, err := kubernetes.GetClientInNamespace(e.env.Config, namespace) + if err != nil { + panic(err) + } + return c } -func (e *Environment) Client(namespace string) client.Client { - return client.NewNamespacedClient(e.intClient, namespace) +func (e *Environment) RestConfig() *rest.Config { + return e.env.Config +} + +func (e *Environment) RawClient() client.Client { + return e.client } -func (e *Environment) start() { +func (e *Environment) Start() { cfg, err := e.env.Start() if err != nil { panic(err) } + k8sClient, err := kubernetes.GetClient(cfg) + if err != nil { + panic(err) + } + + if err := cassdcapi.AddToScheme(k8sClient.Scheme()); err != nil { + panic(err) + } + if err := cassdcapi.AddToScheme(scheme.Scheme); err != nil { panic(err) } @@ -84,15 +101,14 @@ func (e *Environment) start() { //+kubebuilder:scaffold:scheme - k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) - if err != nil { - panic(err) - } - - e.intClient = k8sClient + e.client = k8sClient + // e.Kubeconfig, err = CreateKubeconfigFileForRestConfig(e.env.Config) + // if err != nil { + // panic(err) + // } } -func (e *Environment) stop() { +func (e *Environment) Stop() { e.cancelManager() if err := e.env.Stop(); err != nil { panic(err) @@ -101,13 +117,45 @@ func (e *Environment) stop() { func (e *Environment) CreateNamespace(t *testing.T) string { namespace := strings.ToLower(t.Name()) - if err := kubernetes.CreateNamespaceIfNotExists(e.intClient, namespace); err != nil { + if err := kubernetes.CreateNamespaceIfNotExists(e.client, namespace); err != nil { t.FailNow() } return namespace } -func (e *Environment) RestConfig() *rest.Config { - return e.env.Config +func (e *Environment) GetKubeconfig() ([]byte, error) { + clientConfig, err := CreateKubeconfigFileForRestConfig(e.env.Config) + if err != nil { + return nil, err + } + return clientcmd.Write(clientConfig) +} + +func CreateKubeconfigFileForRestConfig(restConfig *rest.Config) (clientcmdapi.Config, error) { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + AuthInfo: "default-user", + } + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos["default-user"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authinfos, + } + + return clientConfig, nil } diff --git a/internal/envtest/multi_envtest.go b/internal/envtest/multi_envtest.go new file mode 100644 index 0000000..e8da462 --- /dev/null +++ b/internal/envtest/multi_envtest.go @@ -0,0 +1,26 @@ +package envtest + +import ( + "testing" + + ctrl "sigs.k8s.io/controller-runtime" +) + +type MultiK8sEnvironment []*Environment + +func RunMulti(m *testing.M, setupFunc func(e *MultiK8sEnvironment), numClusters int) (code int) { + e := make(MultiK8sEnvironment, numClusters) + ctx := ctrl.SetupSignalHandler() + for i := 0; i < numClusters; i++ { + e[i] = NewEnvironment(ctx) + e[i].Start() + } + defer func() { + for i := 0; i < numClusters; i++ { + e[i].Stop() + } + }() + setupFunc(&e) + exitCode := m.Run() + return exitCode +} diff --git a/internal/envtest/root_dir.go b/internal/envtest/root_dir.go new file mode 100644 index 0000000..320fa88 --- /dev/null +++ b/internal/envtest/root_dir.go @@ -0,0 +1,12 @@ +package envtest + +import ( + "path/filepath" + "runtime" +) + +// https://stackoverflow.com/questions/31873396/is-it-possible-to-get-the-current-root-of-package-structure-as-a-string-in-golan +func RootDir() string { + _, b, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(b), "../..") +} diff --git a/pkg/helmutil/crds_test.go b/pkg/helmutil/crds_test.go index 9c554b6..017767b 100644 --- a/pkg/helmutil/crds_test.go +++ b/pkg/helmutil/crds_test.go @@ -20,7 +20,7 @@ func TestUpgradingCRDs(t *testing.T) { chartNames := []string{"cass-operator"} for _, chartName := range chartNames { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) require.NoError(cleanCache("k8ssandra", chartName)) // creating new upgrader diff --git a/pkg/registration/get_client_from_kubeconfig.go b/pkg/registration/get_client_from_kubeconfig.go new file mode 100644 index 0000000..93bd9fe --- /dev/null +++ b/pkg/registration/get_client_from_kubeconfig.go @@ -0,0 +1,79 @@ +package registration + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetClient(configFileLocation string, contextName string) (client.Client, error) { + clientConfig, err := clientcmd.LoadFromFile(GetKubeconfigFileLocation(configFileLocation)) + if err != nil { + return nil, err + } + var restConfig *rest.Config + if contextName == "" { + restConfig, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + panic(err) + } + return client.New(restConfig, client.Options{}) + } + + context, found := clientConfig.Contexts[contextName] + if !found { + panic(errors.New(fmt.Sprint("context not found in supplied kubeconfig ", "contextName: ", contextName, " configFileLocation: ", GetKubeconfigFileLocation(configFileLocation)))) + } + overrides := &clientcmd.ConfigOverrides{ + Context: *context, + ClusterInfo: *clientConfig.Clusters[context.Cluster], + AuthInfo: *clientConfig.AuthInfos[context.AuthInfo], + } + + cConfig := clientcmd.NewNonInteractiveClientConfig(*clientConfig, contextName, overrides, clientcmd.NewDefaultClientConfigLoadingRules()) + restConfig, err = cConfig.ClientConfig() + + if err != nil { + panic(err) + } + return client.New(restConfig, client.Options{}) +} + +func GetKubeconfigFileLocation(location string) string { + if location != "" { + return location + } else if kubeconfigEnvVar, found := os.LookupEnv("KUBECONFIG"); found { + return kubeconfigEnvVar + } else { + homeDir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + return filepath.Join(homeDir, ".kube", "config") + } +} + +func KubeconfigToHost(configFileLocation string, contextName string) (string, error) { + clientConfig, err := clientcmd.LoadFromFile(GetKubeconfigFileLocation(configFileLocation)) + if err != nil { + return "", err + } + if contextName == "" { + restConfig, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return "", err + } + return restConfig.Host, nil + } + + context, found := clientConfig.Contexts[contextName] + if !found { + panic(errors.New(fmt.Sprint("context not found in supplied kubeconfig ", "contextName: ", contextName, " configFileLocation: ", GetKubeconfigFileLocation(configFileLocation)))) + } + return clientConfig.Clusters[context.Cluster].Server, nil +} diff --git a/pkg/registration/sanitize_strings.go b/pkg/registration/sanitize_strings.go new file mode 100644 index 0000000..85e66d8 --- /dev/null +++ b/pkg/registration/sanitize_strings.go @@ -0,0 +1,25 @@ +package registration + +import ( + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" +) + +var dns1035LabelFmt = "[a-z]([-a-z0-9]*[a-z0-9])?" +var dns1035LabelRegexp = regexp.MustCompile(dns1035LabelFmt) + +func CleanupForKubernetes(input string) string { + if len(validation.IsDNS1035Label(input)) > 0 { + r := dns1035LabelRegexp + + // Invalid domain name, Kubernetes will reject this. Try to modify it to a suitable string + input = strings.ToLower(input) + input = strings.ReplaceAll(input, "_", "-") + validParts := r.FindAllString(input, -1) + return strings.Join(validParts, "") + } + + return input +} diff --git a/pkg/registration/token_to_kubeconfig.go b/pkg/registration/token_to_kubeconfig.go new file mode 100644 index 0000000..fb73064 --- /dev/null +++ b/pkg/registration/token_to_kubeconfig.go @@ -0,0 +1,37 @@ +package registration + +import ( + "errors" + + corev1 "k8s.io/api/core/v1" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +func TokenToKubeconfig(s corev1.Secret, server, destinationName string) (clientcmdapi.Config, error) { + caData, foundCa := s.Data["ca.crt"] + tokenData, foundToken := s.Data["token"] + if !foundCa || !foundToken { + return clientcmdapi.Config{}, errors.New("missing required data in secret") + } + + return clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + destinationName: { + Server: server, + CertificateAuthorityData: caData, + }, + }, + AuthInfos: map[string]*clientcmdapi.AuthInfo{ + destinationName: { + Token: string(tokenData), + }, + }, + Contexts: map[string]*clientcmdapi.Context{ + destinationName: { + Cluster: destinationName, + AuthInfo: destinationName, + }, + }, + CurrentContext: destinationName, + }, nil +} diff --git a/pkg/tasks/create_test.go b/pkg/tasks/create_test.go index f5f0746..1b35a51 100644 --- a/pkg/tasks/create_test.go +++ b/pkg/tasks/create_test.go @@ -6,6 +6,7 @@ import ( cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" controlapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" + k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" "github.com/stretchr/testify/assert" "github.com/k8ssandra/k8ssandra-client/pkg/tasks" @@ -13,11 +14,14 @@ import ( func TestCreateRestartTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" rackName := "rack1" + if err := controlapi.AddToScheme(kubeClient.Scheme()); err != nil { + panic(err) + } task, err := tasks.CreateRestartTask(context.Background(), kubeClient, dc, rackName) @@ -28,22 +32,23 @@ func TestCreateRestartTask(t *testing.T) { func TestCreateClusterRestartTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" rackName := "rack1" - + err := k8ssandrataskapi.AddToScheme(kubeClient.Scheme()) + assert.NoError(t, err) task, err := tasks.CreateClusterRestartTask(context.Background(), kubeClient, namespace, cluster, dcName, rackName) - assert.NoError(t, err) + assert.NotNil(t, task) assert.Equal(t, controlapi.CommandRestart, task.Spec.Template.Jobs[0].Command) } func TestCreateReplaceTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -61,7 +66,7 @@ func TestCreateReplaceTask(t *testing.T) { func TestCreateClusterReplaceTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -79,7 +84,7 @@ func TestCreateClusterReplaceTask(t *testing.T) { func TestCreateFlushTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -95,7 +100,7 @@ func TestCreateFlushTask(t *testing.T) { func TestCreateClusterFlushTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -111,7 +116,7 @@ func TestCreateClusterFlushTask(t *testing.T) { func TestCreateCleanupTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -127,7 +132,7 @@ func TestCreateCleanupTask(t *testing.T) { func TestCreateClusterCleanupTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -143,7 +148,7 @@ func TestCreateClusterCleanupTask(t *testing.T) { func TestCreateUpgradeSSTablesTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -159,7 +164,7 @@ func TestCreateUpgradeSSTablesTask(t *testing.T) { func TestCreateClusterUpgradeSSTablesTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -175,7 +180,7 @@ func TestCreateClusterUpgradeSSTablesTask(t *testing.T) { func TestCreateScrubTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -191,7 +196,7 @@ func TestCreateScrubTask(t *testing.T) { func TestCreateClusterScrubTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -207,7 +212,7 @@ func TestCreateClusterScrubTask(t *testing.T) { func TestCreateCompactionTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -235,7 +240,7 @@ func TestCreateCompactionTask(t *testing.T) { func TestCreateClusterCompactionTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -259,7 +264,7 @@ func TestCreateClusterCompactionTask(t *testing.T) { func TestCreateGCTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -275,7 +280,7 @@ func TestCreateGCTask(t *testing.T) { func TestCreateClusterGCTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -291,7 +296,7 @@ func TestCreateClusterGCTask(t *testing.T) { func TestCreateRebuildTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -312,7 +317,7 @@ func TestCreateRebuildTask(t *testing.T) { func TestCreateClusterRebuildTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -332,7 +337,7 @@ func TestCreateClusterRebuildTask(t *testing.T) { func TestCreateTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) command := controlapi.CommandRestart dc := &cassdcapi.CassandraDatacenter{} @@ -347,7 +352,7 @@ func TestCreateTask(t *testing.T) { func TestCreateTaskLongName(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) command := controlapi.CommandRestart dc := &cassdcapi.CassandraDatacenter{} @@ -362,7 +367,7 @@ func TestCreateTaskLongName(t *testing.T) { func TestCreateClusterWideTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "" diff --git a/testfiles/crd/apiextensions.k8s.io_v1_customresourcedefinition_clientconfigs.config.k8ssandra.io.yaml b/testfiles/crd/apiextensions.k8s.io_v1_customresourcedefinition_clientconfigs.config.k8ssandra.io.yaml new file mode 100644 index 0000000..f03bb0a --- /dev/null +++ b/testfiles/crd/apiextensions.k8s.io_v1_customresourcedefinition_clientconfigs.config.k8ssandra.io.yaml @@ -0,0 +1,61 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: clientconfigs.config.k8ssandra.io +spec: + group: config.k8ssandra.io + names: + kind: ClientConfig + listKind: ClientConfigList + plural: clientconfigs + singular: clientconfig + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ClientConfig is the Schema for the kubeconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClientConfigSpec defines the desired state of KubeConfig + properties: + contextName: + description: ContextName allows to override the object name for context-name. + If not set, the ClientConfig.Name is used as context name + type: string + kubeConfigSecret: + description: |- + KubeConfigSecret should reference an existing secret; the actual configuration will be read from + this secret's "kubeconfig" key. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + type: object + served: true + storage: true