From 705a2cd9ee6443efe20e7574116ab2bef32f9a6d Mon Sep 17 00:00:00 2001 From: Miles Garnsey Date: Thu, 2 May 2024 12:15:34 +1000 Subject: [PATCH] Move the registration work into k8ssandra-client (WIP) More error handling, more defaults. Make some flags persistent. Error out if you try to register a cluster to itself. Remove calls to t.Fatal() as per Micke's request. Micke's feedback, fix up require so it doesn't require *testing.T, ensure IsNotFound is checked prior to Create. Resolve Micke's issues with GetClient. Fix unecessary *testing.T. Rename cmd init func to SetupRegisterClusterCmd and clean up other mistakes. Use envtest.RootDir() where possible, Make client private in Environment. Make sure whole test directory gets cleaned up. Ensure the build directory is created if it does not exist in tests. Remove reconcile.Result Clean up handling of error when src and dest context are the same. destination name from source context name as per Alex' request. Ensure the default context is set on the registration kubeconfig secret. Use sanitized src context name as default secret and clientConfig names. Make sure any isnotFound error doesn't stall the process. Line up all context names across clientConfig and kubeconfig in secret. --- cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go | 2 + cmd/kubectl-k8ssandra/register/command.go | 80 +++++++++ cmd/kubectl-k8ssandra/register/errors.go | 17 ++ .../register/register_test.go | 159 ++++++++++++++++++ .../register/registration.go | 147 ++++++++++++++++ cmd/kubectl-k8ssandra/register/suite_test.go | 19 +++ internal/envtest/envtest.go | 104 +++++++++--- internal/envtest/multi_envtest.go | 26 +++ internal/envtest/root_dir.go | 12 ++ pkg/helmutil/crds_test.go | 2 +- .../get_client_from_kubeconfig.go | 79 +++++++++ pkg/registration/sanitize_strings.go | 25 +++ pkg/registration/token_to_kubeconfig.go | 37 ++++ pkg/tasks/create_test.go | 51 +++--- ...ion_clientconfigs.config.k8ssandra.io.yaml | 61 +++++++ 15 files changed, 769 insertions(+), 52 deletions(-) create mode 100644 cmd/kubectl-k8ssandra/register/command.go create mode 100644 cmd/kubectl-k8ssandra/register/errors.go create mode 100644 cmd/kubectl-k8ssandra/register/register_test.go create mode 100644 cmd/kubectl-k8ssandra/register/registration.go create mode 100644 cmd/kubectl-k8ssandra/register/suite_test.go create mode 100644 internal/envtest/multi_envtest.go create mode 100644 internal/envtest/root_dir.go create mode 100644 pkg/registration/get_client_from_kubeconfig.go create mode 100644 pkg/registration/sanitize_strings.go create mode 100644 pkg/registration/token_to_kubeconfig.go create mode 100644 testfiles/crd/apiextensions.k8s.io_v1_customresourcedefinition_clientconfigs.config.k8ssandra.io.yaml diff --git a/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go b/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go index d734803..4631978 100644 --- a/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go +++ b/cmd/kubectl-k8ssandra/k8ssandra/k8ssandra.go @@ -11,6 +11,7 @@ import ( "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/config" "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/helm" "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/operate" + "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/register" "github.com/k8ssandra/k8ssandra-client/cmd/kubectl-k8ssandra/users" "github.com/spf13/cobra" @@ -53,6 +54,7 @@ func NewCmd(streams genericclioptions.IOStreams) *cobra.Command { // cmd.AddCommand(migrate.NewInstallCmd(streams)) cmd.AddCommand(config.NewCmd(streams)) cmd.AddCommand(helm.NewHelmCmd(streams)) + register.SetupRegisterClusterCmd(cmd, streams) // cmd.Flags().BoolVar(&o.listNamespaces, "list", o.listNamespaces, "if true, print the list of all namespaces in the current KUBECONFIG") o.configFlags.AddFlags(cmd.Flags()) diff --git a/cmd/kubectl-k8ssandra/register/command.go b/cmd/kubectl-k8ssandra/register/command.go new file mode 100644 index 0000000..95de8a5 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/command.go @@ -0,0 +1,80 @@ +package register + +import ( + "fmt" + + "github.com/charmbracelet/log" + "github.com/k8ssandra/k8ssandra-client/pkg/registration" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +var RegisterClusterCmd = &cobra.Command{ + Use: "register [flags]", + Short: "register a data plane into the control plane.", + Long: `register creates a ServiceAccount on a source cluster, copies its credentials and then creates a secret containing them on the destination cluster. It then also creates a ClientConfig on the destination cluster to reference the secret.`, + Run: entrypoint, +} + +func SetupRegisterClusterCmd(cmd *cobra.Command, streams genericclioptions.IOStreams) { + RegisterClusterCmd.Flags().String("source-kubeconfig", + "", + "path to source cluster's kubeconfig file - defaults to KUBECONFIG then ~/.kube/config") + RegisterClusterCmd.Flags().String("dest-kubeconfig", + "", + "path to destination cluster's kubeconfig file - defaults to KUBECONFIG then ~/.kube/config") + RegisterClusterCmd.Flags().String("source-context", "", "context name for source cluster") + RegisterClusterCmd.Flags().String("dest-context", "", "context name for destination cluster") + RegisterClusterCmd.Flags().String("source-namespace", "k8ssandra-operator", "namespace containing service account for source cluster") + RegisterClusterCmd.Flags().String("dest-namespace", "k8ssandra-operator", "namespace where secret and clientConfig will be created on destination cluster") + RegisterClusterCmd.Flags().String("serviceaccount-name", "k8ssandra-operator", "serviceaccount name for destination cluster") + RegisterClusterCmd.Flags().String("destination-name", "", "name for remote clientConfig and secret on destination cluster") + + if err := RegisterClusterCmd.MarkFlagRequired("source-context"); err != nil { + panic(err) + + } + if err := RegisterClusterCmd.MarkFlagRequired("dest-context"); err != nil { + panic(err) + } + cmd.AddCommand(RegisterClusterCmd) +} + +func entrypoint(cmd *cobra.Command, args []string) { + executor := NewRegistrationExecutorFromRegisterClusterCmd(*cmd) + + for i := 0; i < 30; i++ { + res := executor.RegisterCluster() + switch v := res.(type) { + case RetryableError: + log.Info("Registration continuing", "msg", v.Error()) + continue + case nil: + log.Info("Registration completed successfully") + return + case NonRecoverableError: + panic(fmt.Sprintf("Registration failed: %s", v.Error())) + } + } + fmt.Println("Registration failed - retries exceeded") +} + +func NewRegistrationExecutorFromRegisterClusterCmd(cmd cobra.Command) *RegistrationExecutor { + + destName := cmd.Flag("destination-name").Value.String() + srcContext := cmd.Flag("source-context").Value.String() + if destName == "" { + destName = registration.CleanupForKubernetes(srcContext) + } + return &RegistrationExecutor{ + SourceKubeconfig: cmd.Flag("source-kubeconfig").Value.String(), + DestKubeconfig: cmd.Flag("dest-kubeconfig").Value.String(), + SourceContext: srcContext, + DestContext: cmd.Flag("dest-context").Value.String(), + SourceNamespace: cmd.Flag("source-namespace").Value.String(), + DestNamespace: cmd.Flag("dest-namespace").Value.String(), + ServiceAccount: cmd.Flag("serviceaccount-name").Value.String(), + Context: cmd.Context(), + DestinationName: destName, + } +} diff --git a/cmd/kubectl-k8ssandra/register/errors.go b/cmd/kubectl-k8ssandra/register/errors.go new file mode 100644 index 0000000..1a86f87 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/errors.go @@ -0,0 +1,17 @@ +package register + +type RetryableError struct { + Message string +} + +func (e RetryableError) Error() string { + return e.Message +} + +type NonRecoverableError struct { + Message string +} + +func (e NonRecoverableError) Error() string { + return e.Message +} diff --git a/cmd/kubectl-k8ssandra/register/register_test.go b/cmd/kubectl-k8ssandra/register/register_test.go new file mode 100644 index 0000000..56f1cd8 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/register_test.go @@ -0,0 +1,159 @@ +package register + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + configapi "github.com/k8ssandra/k8ssandra-operator/apis/config/v1beta1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestRegister(t *testing.T) { + require := require.New(t) + client1 := (*multiEnv)[0].GetClientInNamespace("source-namespace") + client2 := (*multiEnv)[1].GetClientInNamespace("dest-namespace") + require.NoError(client1.Create((*multiEnv)[0].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "source-namespace"}})) + require.NoError(client2.Create((*multiEnv)[1].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dest-namespace"}})) + + testDir, err := os.MkdirTemp("", "k8ssandra-operator-test-****") + require.NoError(err) + // buildDir := filepath.Join(envtest.RootDir(), "build") + // testDir := filepath.Join(buildDir, time.Now()) + + // if _, err := os.Stat(testDir); os.IsNotExist(err) { + // err := os.MkdirAll(testDir, os.ModePerm) + // require.NoError(err) + // } else if err != nil { + // require.NoError(err) + // } + t.Cleanup(func() { + require.NoError(os.RemoveAll(testDir)) + }) + + kc1, err := (*multiEnv)[0].GetKubeconfig() + require.NoError(err) + f1, err := os.Create(testDir + "/kubeconfig1") + require.NoError(err) + t.Cleanup(func() { + require.NoError(f1.Close()) + }) + _, err = f1.Write(kc1) + require.NoError(err) + + f2, err := os.Create(testDir + "/kubeconfig2") + require.NoError(err) + t.Cleanup(func() { + require.NoError(f2.Close()) + }) + + kc2, err := (*multiEnv)[1].GetKubeconfig() + require.NoError(err) + _, err = f2.Write(kc2) + require.NoError(err) + + ex := RegistrationExecutor{ + SourceKubeconfig: testDir + "/kubeconfig1", + DestKubeconfig: testDir + "/kubeconfig2", + SourceContext: "default-context", + DestContext: "default-context", + SourceNamespace: "source-namespace", + DestNamespace: "dest-namespace", + ServiceAccount: "k8ssandra-operator", + Context: context.TODO(), + DestinationName: "test-destination", + } + ctx := context.Background() + + require.Eventually(func() bool { + res := ex.RegisterCluster() + switch v := res.(type) { + case RetryableError: + if res.Error() == "no secret found for service account k8ssandra-operator" { + return true + } + case nil: + return true + case NonRecoverableError: + panic(fmt.Sprintf("Registration failed: %s", v.Error())) + } + return false + }, time.Second*30, time.Second*5) + + // This relies on a controller that is not running in the envtest. + + desiredSaSecret := &corev1.Secret{} + require.NoError(client1.Get(context.Background(), client.ObjectKey{Name: "k8ssandra-operator-secret", Namespace: "source-namespace"}, desiredSaSecret)) + patch := client.MergeFrom(desiredSaSecret.DeepCopy()) + desiredSaSecret.Data = map[string][]byte{ + "token": []byte("test-token"), + "ca.crt": []byte("test-ca"), + } + require.NoError(client1.Patch(ctx, desiredSaSecret, patch)) + + desiredSa := &corev1.ServiceAccount{} + require.NoError(client1.Get( + context.Background(), + client.ObjectKey{Name: "k8ssandra-operator", Namespace: "source-namespace"}, + desiredSa)) + + patch = client.MergeFrom(desiredSa.DeepCopy()) + desiredSa.Secrets = []corev1.ObjectReference{ + { + Name: "k8ssandra-operator-secret", + }, + } + require.NoError(client1.Patch(ctx, desiredSa, patch)) + + // Continue reconciliation + + require.Eventually(func() bool { + res := ex.RegisterCluster() + return res == nil + }, time.Second*300, time.Second*1) + + if err := configapi.AddToScheme(client2.Scheme()); err != nil { + require.NoError(err) + } + destSecret := &corev1.Secret{} + require.Eventually(func() bool { + err = client2.Get(ctx, + client.ObjectKey{Name: "test-destination", Namespace: "dest-namespace"}, destSecret) + if err != nil { + t.Log("didn't find dest secret") + return false + } + clientConfig := &configapi.ClientConfig{} + err = client2.Get(ctx, + client.ObjectKey{Name: "test-destination", Namespace: "dest-namespace"}, clientConfig) + if err != nil { + t.Log("didn't find dest client config") + return false + } + return err == nil + }, time.Second*60, time.Second*5) + + destKubeconfig := ClientConfigFromSecret(destSecret) + require.Equal( + desiredSaSecret.Data["ca.crt"], + destKubeconfig.Clusters["test-destination"].CertificateAuthorityData) + + require.Equal( + string(desiredSaSecret.Data["token"]), + destKubeconfig.AuthInfos["test-destination"].Token) +} + +func ClientConfigFromSecret(s *corev1.Secret) clientcmdapi.Config { + out, err := clientcmd.Load(s.Data["kubeconfig"]) + if err != nil { + panic(err) + } + return *out +} diff --git a/cmd/kubectl-k8ssandra/register/registration.go b/cmd/kubectl-k8ssandra/register/registration.go new file mode 100644 index 0000000..71f68a6 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/registration.go @@ -0,0 +1,147 @@ +package register + +import ( + "context" + "fmt" + + "github.com/charmbracelet/log" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/k8ssandra/k8ssandra-client/pkg/registration" + configapi "github.com/k8ssandra/k8ssandra-operator/apis/config/v1beta1" +) + +type RegistrationExecutor struct { + DestinationName string + SourceKubeconfig string + DestKubeconfig string + SourceContext string + DestContext string + SourceNamespace string + DestNamespace string + ServiceAccount string + Context context.Context +} + +func getDefaultSecret(saNamespace, saName string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName + "-secret", + Namespace: saNamespace, + Annotations: map[string]string{ + "kubernetes.io/service-account.name": saName, + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + } +} + +func getDefaultServiceAccount(saName, saNamespace string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: saNamespace, + }, + } +} + +func (e *RegistrationExecutor) RegisterCluster() error { + log.Printf("Registering cluster from %s Context: %s to %s Context: %s", + registration.GetKubeconfigFileLocation(e.SourceKubeconfig), e.SourceContext, + registration.GetKubeconfigFileLocation(e.DestKubeconfig), e.DestContext, + ) + if e.SourceContext == e.DestContext && e.SourceKubeconfig == e.DestKubeconfig { + return NonRecoverableError{Message: "source and destination context and kubeconfig are the same, you should not register the same cluster to itself. Reference it by leaving the k8sContext field blank instead"} + } + srcClient, err := registration.GetClient(e.SourceKubeconfig, e.SourceContext) + if err != nil { + return RetryableError{Message: err.Error()} + } + destClient, err := registration.GetClient(e.DestKubeconfig, e.DestContext) + if err != nil { + return RetryableError{Message: err.Error()} + } + // Get ServiceAccount + serviceAccount := &corev1.ServiceAccount{} + if err := srcClient.Get(e.Context, client.ObjectKey{Name: e.ServiceAccount, Namespace: e.SourceNamespace}, serviceAccount); err != nil { + if apierrors.IsNotFound(err) { + if err := srcClient.Create(e.Context, getDefaultServiceAccount(e.ServiceAccount, e.SourceNamespace)); err != nil { + return RetryableError{Message: err.Error()} + } + } + return RetryableError{Message: err.Error()} + } + // Get a secret in this namespace which holds the service account token + secretsList := &corev1.SecretList{} + if err := srcClient.List(e.Context, secretsList, client.InNamespace(e.SourceNamespace)); err != nil { + return RetryableError{Message: err.Error()} + } + var secret *corev1.Secret + for _, s := range secretsList.Items { + if s.Annotations["kubernetes.io/service-account.name"] == e.ServiceAccount && s.Type == corev1.SecretTypeServiceAccountToken { + secret = &s + break + } + } + if secret == nil { + secret = getDefaultSecret(e.SourceNamespace, e.ServiceAccount) + if err := srcClient.Create(e.Context, secret); err != nil { + return RetryableError{Message: err.Error()} + } + return RetryableError{Message: fmt.Sprintf("no secret found for service account %s", e.ServiceAccount)} + } + + // Create Secret on destination cluster + host, err := registration.KubeconfigToHost(e.SourceKubeconfig, e.SourceContext) + if err != nil { + return RetryableError{Message: err.Error()} + } + saConfig, err := registration.TokenToKubeconfig(*secret, host, e.DestinationName) + if err != nil { + return RetryableError{fmt.Sprintf("error converting token to kubeconfig: %s, secret: %#v", err.Error(), secret)} + } + secretData, err := clientcmd.Write(saConfig) + if err != nil { + return RetryableError{Message: err.Error()} + } + destSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: e.DestinationName, + Namespace: e.DestNamespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "kubeconfig": secretData, + }, + } + if err := destClient.Create(e.Context, &destSecret); err != nil && !errors.IsAlreadyExists(err) { + return RetryableError{fmt.Sprintf("error creating secret. err: %s sa %s", err, e.ServiceAccount)} + } + + // Create ClientConfig on destination cluster + if err := configapi.AddToScheme(destClient.Scheme()); err != nil { + return RetryableError{Message: err.Error()} + } + destClientConfig := configapi.ClientConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: e.DestinationName, + Namespace: e.DestNamespace, + }, + Spec: configapi.ClientConfigSpec{ + KubeConfigSecret: corev1.LocalObjectReference{ + Name: e.DestinationName, + }, + ContextName: e.DestinationName, + }, + } + if err := destClient.Create(e.Context, &destClientConfig); err != nil && !errors.IsAlreadyExists(err) { + return RetryableError{Message: err.Error()} + } + return nil +} diff --git a/cmd/kubectl-k8ssandra/register/suite_test.go b/cmd/kubectl-k8ssandra/register/suite_test.go new file mode 100644 index 0000000..0211a89 --- /dev/null +++ b/cmd/kubectl-k8ssandra/register/suite_test.go @@ -0,0 +1,19 @@ +package register + +import ( + "os" + "testing" + + "github.com/k8ssandra/k8ssandra-client/internal/envtest" +) + +var ( + multiEnv *envtest.MultiK8sEnvironment +) + +func TestMain(m *testing.M) { + // metrics.DefaultBindAddress = "0" This no longer appears to exist... + os.Exit(envtest.RunMulti(m, func(e *envtest.MultiK8sEnvironment) { + multiEnv = e + }, 2)) +} diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index e04bf5d..120c290 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -3,17 +3,18 @@ package envtest import ( "context" "path/filepath" - "runtime" "strings" "testing" cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" controlapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" + "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" - apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/kubectl/pkg/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,51 +22,67 @@ import ( ) func Run(m *testing.M, setupFunc func(e *Environment)) (code int) { - env := NewEnvironment() - env.start() + ctx := ctrl.SetupSignalHandler() + env := NewEnvironment(ctx) + env.Start() setupFunc(env) exitCode := m.Run() - env.stop() + env.Stop() return exitCode } type Environment struct { - intClient client.Client + client client.Client env *envtest.Environment cancelManager context.CancelFunc Context context.Context } -func NewEnvironment() *Environment { +func NewEnvironment(ctx context.Context) *Environment { env := &Environment{} env.env = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join(RootDir(), "testfiles", "crd")}, + CRDDirectoryPaths: []string{ + filepath.Join(RootDir(), "testfiles", "crd"), + }, ErrorIfCRDPathMissing: true, } - - ctx := ctrl.SetupSignalHandler() ctx, cancel := context.WithCancel(ctx) env.Context = ctx env.cancelManager = cancel return env } -// https://stackoverflow.com/questions/31873396/is-it-possible-to-get-the-current-root-of-package-structure-as-a-string-in-golan -func RootDir() string { - _, b, _, _ := runtime.Caller(0) - return filepath.Join(filepath.Dir(b), "../..") +func (e *Environment) GetClientInNamespace(namespace string) client.Client { + c, err := kubernetes.GetClientInNamespace(e.env.Config, namespace) + if err != nil { + panic(err) + } + return c } -func (e *Environment) Client(namespace string) client.Client { - return client.NewNamespacedClient(e.intClient, namespace) +func (e *Environment) RestConfig() *rest.Config { + return e.env.Config +} + +func (e *Environment) RawClient() client.Client { + return e.client } -func (e *Environment) start() { +func (e *Environment) Start() { cfg, err := e.env.Start() if err != nil { panic(err) } + k8sClient, err := kubernetes.GetClient(cfg) + if err != nil { + panic(err) + } + + if err := cassdcapi.AddToScheme(k8sClient.Scheme()); err != nil { + panic(err) + } + if err := cassdcapi.AddToScheme(scheme.Scheme); err != nil { panic(err) } @@ -84,15 +101,14 @@ func (e *Environment) start() { //+kubebuilder:scaffold:scheme - k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) - if err != nil { - panic(err) - } - - e.intClient = k8sClient + e.client = k8sClient + // e.Kubeconfig, err = CreateKubeconfigFileForRestConfig(e.env.Config) + // if err != nil { + // panic(err) + // } } -func (e *Environment) stop() { +func (e *Environment) Stop() { e.cancelManager() if err := e.env.Stop(); err != nil { panic(err) @@ -101,13 +117,45 @@ func (e *Environment) stop() { func (e *Environment) CreateNamespace(t *testing.T) string { namespace := strings.ToLower(t.Name()) - if err := kubernetes.CreateNamespaceIfNotExists(e.intClient, namespace); err != nil { + if err := kubernetes.CreateNamespaceIfNotExists(e.client, namespace); err != nil { t.FailNow() } return namespace } -func (e *Environment) RestConfig() *rest.Config { - return e.env.Config +func (e *Environment) GetKubeconfig() ([]byte, error) { + clientConfig, err := CreateKubeconfigFileForRestConfig(e.env.Config) + if err != nil { + return nil, err + } + return clientcmd.Write(clientConfig) +} + +func CreateKubeconfigFileForRestConfig(restConfig *rest.Config) (clientcmdapi.Config, error) { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + AuthInfo: "default-user", + } + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos["default-user"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authinfos, + } + + return clientConfig, nil } diff --git a/internal/envtest/multi_envtest.go b/internal/envtest/multi_envtest.go new file mode 100644 index 0000000..e8da462 --- /dev/null +++ b/internal/envtest/multi_envtest.go @@ -0,0 +1,26 @@ +package envtest + +import ( + "testing" + + ctrl "sigs.k8s.io/controller-runtime" +) + +type MultiK8sEnvironment []*Environment + +func RunMulti(m *testing.M, setupFunc func(e *MultiK8sEnvironment), numClusters int) (code int) { + e := make(MultiK8sEnvironment, numClusters) + ctx := ctrl.SetupSignalHandler() + for i := 0; i < numClusters; i++ { + e[i] = NewEnvironment(ctx) + e[i].Start() + } + defer func() { + for i := 0; i < numClusters; i++ { + e[i].Stop() + } + }() + setupFunc(&e) + exitCode := m.Run() + return exitCode +} diff --git a/internal/envtest/root_dir.go b/internal/envtest/root_dir.go new file mode 100644 index 0000000..320fa88 --- /dev/null +++ b/internal/envtest/root_dir.go @@ -0,0 +1,12 @@ +package envtest + +import ( + "path/filepath" + "runtime" +) + +// https://stackoverflow.com/questions/31873396/is-it-possible-to-get-the-current-root-of-package-structure-as-a-string-in-golan +func RootDir() string { + _, b, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(b), "../..") +} diff --git a/pkg/helmutil/crds_test.go b/pkg/helmutil/crds_test.go index 9c554b6..017767b 100644 --- a/pkg/helmutil/crds_test.go +++ b/pkg/helmutil/crds_test.go @@ -20,7 +20,7 @@ func TestUpgradingCRDs(t *testing.T) { chartNames := []string{"cass-operator"} for _, chartName := range chartNames { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) require.NoError(cleanCache("k8ssandra", chartName)) // creating new upgrader diff --git a/pkg/registration/get_client_from_kubeconfig.go b/pkg/registration/get_client_from_kubeconfig.go new file mode 100644 index 0000000..93bd9fe --- /dev/null +++ b/pkg/registration/get_client_from_kubeconfig.go @@ -0,0 +1,79 @@ +package registration + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetClient(configFileLocation string, contextName string) (client.Client, error) { + clientConfig, err := clientcmd.LoadFromFile(GetKubeconfigFileLocation(configFileLocation)) + if err != nil { + return nil, err + } + var restConfig *rest.Config + if contextName == "" { + restConfig, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + panic(err) + } + return client.New(restConfig, client.Options{}) + } + + context, found := clientConfig.Contexts[contextName] + if !found { + panic(errors.New(fmt.Sprint("context not found in supplied kubeconfig ", "contextName: ", contextName, " configFileLocation: ", GetKubeconfigFileLocation(configFileLocation)))) + } + overrides := &clientcmd.ConfigOverrides{ + Context: *context, + ClusterInfo: *clientConfig.Clusters[context.Cluster], + AuthInfo: *clientConfig.AuthInfos[context.AuthInfo], + } + + cConfig := clientcmd.NewNonInteractiveClientConfig(*clientConfig, contextName, overrides, clientcmd.NewDefaultClientConfigLoadingRules()) + restConfig, err = cConfig.ClientConfig() + + if err != nil { + panic(err) + } + return client.New(restConfig, client.Options{}) +} + +func GetKubeconfigFileLocation(location string) string { + if location != "" { + return location + } else if kubeconfigEnvVar, found := os.LookupEnv("KUBECONFIG"); found { + return kubeconfigEnvVar + } else { + homeDir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + return filepath.Join(homeDir, ".kube", "config") + } +} + +func KubeconfigToHost(configFileLocation string, contextName string) (string, error) { + clientConfig, err := clientcmd.LoadFromFile(GetKubeconfigFileLocation(configFileLocation)) + if err != nil { + return "", err + } + if contextName == "" { + restConfig, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return "", err + } + return restConfig.Host, nil + } + + context, found := clientConfig.Contexts[contextName] + if !found { + panic(errors.New(fmt.Sprint("context not found in supplied kubeconfig ", "contextName: ", contextName, " configFileLocation: ", GetKubeconfigFileLocation(configFileLocation)))) + } + return clientConfig.Clusters[context.Cluster].Server, nil +} diff --git a/pkg/registration/sanitize_strings.go b/pkg/registration/sanitize_strings.go new file mode 100644 index 0000000..85e66d8 --- /dev/null +++ b/pkg/registration/sanitize_strings.go @@ -0,0 +1,25 @@ +package registration + +import ( + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" +) + +var dns1035LabelFmt = "[a-z]([-a-z0-9]*[a-z0-9])?" +var dns1035LabelRegexp = regexp.MustCompile(dns1035LabelFmt) + +func CleanupForKubernetes(input string) string { + if len(validation.IsDNS1035Label(input)) > 0 { + r := dns1035LabelRegexp + + // Invalid domain name, Kubernetes will reject this. Try to modify it to a suitable string + input = strings.ToLower(input) + input = strings.ReplaceAll(input, "_", "-") + validParts := r.FindAllString(input, -1) + return strings.Join(validParts, "") + } + + return input +} diff --git a/pkg/registration/token_to_kubeconfig.go b/pkg/registration/token_to_kubeconfig.go new file mode 100644 index 0000000..fb73064 --- /dev/null +++ b/pkg/registration/token_to_kubeconfig.go @@ -0,0 +1,37 @@ +package registration + +import ( + "errors" + + corev1 "k8s.io/api/core/v1" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +func TokenToKubeconfig(s corev1.Secret, server, destinationName string) (clientcmdapi.Config, error) { + caData, foundCa := s.Data["ca.crt"] + tokenData, foundToken := s.Data["token"] + if !foundCa || !foundToken { + return clientcmdapi.Config{}, errors.New("missing required data in secret") + } + + return clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + destinationName: { + Server: server, + CertificateAuthorityData: caData, + }, + }, + AuthInfos: map[string]*clientcmdapi.AuthInfo{ + destinationName: { + Token: string(tokenData), + }, + }, + Contexts: map[string]*clientcmdapi.Context{ + destinationName: { + Cluster: destinationName, + AuthInfo: destinationName, + }, + }, + CurrentContext: destinationName, + }, nil +} diff --git a/pkg/tasks/create_test.go b/pkg/tasks/create_test.go index f5f0746..1b35a51 100644 --- a/pkg/tasks/create_test.go +++ b/pkg/tasks/create_test.go @@ -6,6 +6,7 @@ import ( cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" controlapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" + k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" "github.com/stretchr/testify/assert" "github.com/k8ssandra/k8ssandra-client/pkg/tasks" @@ -13,11 +14,14 @@ import ( func TestCreateRestartTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" rackName := "rack1" + if err := controlapi.AddToScheme(kubeClient.Scheme()); err != nil { + panic(err) + } task, err := tasks.CreateRestartTask(context.Background(), kubeClient, dc, rackName) @@ -28,22 +32,23 @@ func TestCreateRestartTask(t *testing.T) { func TestCreateClusterRestartTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" rackName := "rack1" - + err := k8ssandrataskapi.AddToScheme(kubeClient.Scheme()) + assert.NoError(t, err) task, err := tasks.CreateClusterRestartTask(context.Background(), kubeClient, namespace, cluster, dcName, rackName) - assert.NoError(t, err) + assert.NotNil(t, task) assert.Equal(t, controlapi.CommandRestart, task.Spec.Template.Jobs[0].Command) } func TestCreateReplaceTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -61,7 +66,7 @@ func TestCreateReplaceTask(t *testing.T) { func TestCreateClusterReplaceTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -79,7 +84,7 @@ func TestCreateClusterReplaceTask(t *testing.T) { func TestCreateFlushTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -95,7 +100,7 @@ func TestCreateFlushTask(t *testing.T) { func TestCreateClusterFlushTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -111,7 +116,7 @@ func TestCreateClusterFlushTask(t *testing.T) { func TestCreateCleanupTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -127,7 +132,7 @@ func TestCreateCleanupTask(t *testing.T) { func TestCreateClusterCleanupTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -143,7 +148,7 @@ func TestCreateClusterCleanupTask(t *testing.T) { func TestCreateUpgradeSSTablesTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -159,7 +164,7 @@ func TestCreateUpgradeSSTablesTask(t *testing.T) { func TestCreateClusterUpgradeSSTablesTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -175,7 +180,7 @@ func TestCreateClusterUpgradeSSTablesTask(t *testing.T) { func TestCreateScrubTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -191,7 +196,7 @@ func TestCreateScrubTask(t *testing.T) { func TestCreateClusterScrubTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -207,7 +212,7 @@ func TestCreateClusterScrubTask(t *testing.T) { func TestCreateCompactionTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -235,7 +240,7 @@ func TestCreateCompactionTask(t *testing.T) { func TestCreateClusterCompactionTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -259,7 +264,7 @@ func TestCreateClusterCompactionTask(t *testing.T) { func TestCreateGCTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -275,7 +280,7 @@ func TestCreateGCTask(t *testing.T) { func TestCreateClusterGCTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -291,7 +296,7 @@ func TestCreateClusterGCTask(t *testing.T) { func TestCreateRebuildTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) dc := &cassdcapi.CassandraDatacenter{} dc.Name = "test-dc" @@ -312,7 +317,7 @@ func TestCreateRebuildTask(t *testing.T) { func TestCreateClusterRebuildTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "test-dc" @@ -332,7 +337,7 @@ func TestCreateClusterRebuildTask(t *testing.T) { func TestCreateTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) command := controlapi.CommandRestart dc := &cassdcapi.CassandraDatacenter{} @@ -347,7 +352,7 @@ func TestCreateTask(t *testing.T) { func TestCreateTaskLongName(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) command := controlapi.CommandRestart dc := &cassdcapi.CassandraDatacenter{} @@ -362,7 +367,7 @@ func TestCreateTaskLongName(t *testing.T) { func TestCreateClusterWideTask(t *testing.T) { namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + kubeClient := env.GetClientInNamespace(namespace) cluster := "test-cluster" dcName := "" diff --git a/testfiles/crd/apiextensions.k8s.io_v1_customresourcedefinition_clientconfigs.config.k8ssandra.io.yaml b/testfiles/crd/apiextensions.k8s.io_v1_customresourcedefinition_clientconfigs.config.k8ssandra.io.yaml new file mode 100644 index 0000000..f03bb0a --- /dev/null +++ b/testfiles/crd/apiextensions.k8s.io_v1_customresourcedefinition_clientconfigs.config.k8ssandra.io.yaml @@ -0,0 +1,61 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: clientconfigs.config.k8ssandra.io +spec: + group: config.k8ssandra.io + names: + kind: ClientConfig + listKind: ClientConfigList + plural: clientconfigs + singular: clientconfig + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ClientConfig is the Schema for the kubeconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClientConfigSpec defines the desired state of KubeConfig + properties: + contextName: + description: ContextName allows to override the object name for context-name. + If not set, the ClientConfig.Name is used as context name + type: string + kubeConfigSecret: + description: |- + KubeConfigSecret should reference an existing secret; the actual configuration will be read from + this secret's "kubeconfig" key. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + type: object + served: true + storage: true