diff --git a/Makefile b/Makefile index f5889f5..21e4c7c 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.27.x +ENVTEST_K8S_VERSION = 1.28.x .PHONY: all all: build diff --git a/cmd/kubectl-k8ssandra/helm/crds.go b/cmd/kubectl-k8ssandra/helm/crds.go new file mode 100644 index 0000000..cf23aa5 --- /dev/null +++ b/cmd/kubectl-k8ssandra/helm/crds.go @@ -0,0 +1,122 @@ +package helm + +import ( + "context" + "fmt" + + "github.com/k8ssandra/k8ssandra-client/pkg/helmutil" + "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +var ( + upgraderExample = ` + # update CRDs in the namespace to targetVersion + %[1]s crds --chartName --targetVersion [] + + # update CRDs in the namespace to targetVersion with non-default chartRepo (helm.k8ssandra.io) + %[1]s crds --chartName --targetVersion --chartRepo [] + ` + errNotEnoughParameters = fmt.Errorf("not enough parameters, requires chartName and targetVersion") +) + +type options struct { + configFlags *genericclioptions.ConfigFlags + genericclioptions.IOStreams + namespace string + chartName string + targetVersion string + chartRepo string + repoURL string +} + +func newOptions(streams genericclioptions.IOStreams) *options { + return &options{ + configFlags: genericclioptions.NewConfigFlags(true), + IOStreams: streams, + } +} + +// NewCmd provides a cobra command wrapping cqlShOptions +func NewUpgradeCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := newOptions(streams) + + cmd := &cobra.Command{ + Use: "upgrade [flags]", + Short: "upgrade k8ssandra CRDs to target release version", + Example: fmt.Sprintf(upgraderExample, "kubectl k8ssandra helm crds"), + SilenceUsage: true, + RunE: func(c *cobra.Command, args []string) error { + if err := o.Complete(c, args); err != nil { + return err + } + if err := o.Validate(); err != nil { + return err + } + if err := o.Run(); err != nil { + return err + } + + return nil + }, + } + + fl := cmd.Flags() + fl.StringVar(&o.chartName, "chartName", "", "chartName to upgrade") + fl.StringVar(&o.targetVersion, "targetVersion", "", "targetVersion to upgrade to") + fl.StringVar(&o.chartRepo, "chartRepo", "", "optional chart repository name to override the default (k8ssandra)") + fl.StringVar(&o.repoURL, "repoURL", "", "optional chart repository url to override the default (helm.k8ssandra.io)") + o.configFlags.AddFlags(fl) + + return cmd +} + +// Complete parses the arguments and necessary flags to options +func (c *options) Complete(cmd *cobra.Command, args []string) error { + var err error + if len(args) < 2 { + return errNotEnoughParameters + } + + if c.repoURL == "" { + c.repoURL = helmutil.StableK8ssandraRepoURL + } + + if c.chartRepo == "" { + c.chartRepo = helmutil.K8ssandraRepoName + } + + c.targetVersion = args[0] + c.namespace, _, err = c.configFlags.ToRawKubeConfigLoader().Namespace() + return err +} + +// Validate ensures that all required arguments and flag values are provided +func (c *options) Validate() error { + // TODO Validate that the targetVersion is valid + return nil +} + +// Run removes the finalizers for a release X in the given namespace +func (c *options) Run() error { + restConfig, err := c.configFlags.ToRESTConfig() + if err != nil { + return err + } + + kubeClient, err := kubernetes.GetClientInNamespace(restConfig, c.namespace) + if err != nil { + return err + } + + ctx := context.Background() + + upgrader, err := helmutil.NewUpgrader(kubeClient, c.chartRepo, c.repoURL, c.chartName) + if err != nil { + return err + } + + _, err = upgrader.Upgrade(ctx, c.targetVersion) + return err +} diff --git a/cmd/kubectl-k8ssandra/helm/helm.go b/cmd/kubectl-k8ssandra/helm/helm.go new file mode 100644 index 0000000..8d958a3 --- /dev/null +++ b/cmd/kubectl-k8ssandra/helm/helm.go @@ -0,0 +1,36 @@ +package helm + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +type ClientOptions struct { + configFlags *genericclioptions.ConfigFlags + genericclioptions.IOStreams +} + +// NewClientOptions provides an instance of NamespaceOptions with default values +func NewHelmOptions(streams genericclioptions.IOStreams) *ClientOptions { + return &ClientOptions{ + configFlags: genericclioptions.NewConfigFlags(true), + IOStreams: streams, + } +} + +// NewCmd provides a cobra command wrapping NamespaceOptions +func NewHelmCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := NewHelmOptions(streams) + + cmd := &cobra.Command{ + Use: "k8ssandra [subcommand] [flags]", + } + + // Add subcommands + cmd.AddCommand(NewUpgradeCmd(streams)) + + // cmd.Flags().BoolVar(&o.listNamespaces, "list", o.listNamespaces, "if true, print the list of all namespaces in the current KUBECONFIG") + o.configFlags.AddFlags(cmd.Flags()) + + return cmd +} diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index ce7d6ba..e04bf5d 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -12,6 +12,8 @@ import ( k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/client-go/rest" "k8s.io/kubectl/pkg/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -76,6 +78,10 @@ func (e *Environment) start() { panic(err) } + if err := apiextensions.AddToScheme(scheme.Scheme); err != nil { + panic(err) + } + //+kubebuilder:scaffold:scheme k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) @@ -101,3 +107,7 @@ func (e *Environment) CreateNamespace(t *testing.T) string { return namespace } + +func (e *Environment) RestConfig() *rest.Config { + return e.env.Config +} diff --git a/pkg/helmutil/crds.go b/pkg/helmutil/crds.go new file mode 100644 index 0000000..516eeb9 --- /dev/null +++ b/pkg/helmutil/crds.go @@ -0,0 +1,154 @@ +package helmutil + +import ( + "bufio" + "bytes" + "context" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + deser "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Upgrader is a utility to update the CRDs in a helm chart's pre-upgrade hook +type Upgrader struct { + client client.Client + repoName string + repoURL string + chartName string +} + +// NewUpgrader returns a new Upgrader client +func NewUpgrader(c client.Client, repoName, repoURL, chartName string) (*Upgrader, error) { + return &Upgrader{ + client: c, + repoName: repoName, + repoURL: repoURL, + chartName: chartName, + }, nil +} + +// Upgrade installs the missing CRDs or updates them if they exists already +func (u *Upgrader) Upgrade(ctx context.Context, targetVersion string) ([]unstructured.Unstructured, error) { + chartDir, err := GetChartTargetDir(u.chartName, targetVersion) + if err != nil { + return nil, err + } + + if _, err := os.Stat(chartDir); os.IsNotExist(err) { + downloadDir, err := DownloadChartRelease(u.repoName, u.repoURL, u.chartName, targetVersion) + if err != nil { + return nil, err + } + + extractDir, err := ExtractChartRelease(downloadDir, u.chartName, targetVersion) + if err != nil { + return nil, err + } + chartDir = extractDir + } + + // defer os.RemoveAll(downloadDir) + + crds := make([]unstructured.Unstructured, 0) + + // For each dir under the charts subdir, check the "crds/" + paths, _ := findCRDDirs(chartDir) + + for _, path := range paths { + err = parseChartCRDs(&crds, path) + if err != nil { + return nil, err + } + } + + for _, obj := range crds { + existingCrd := obj.DeepCopy() + err = u.client.Get(ctx, client.ObjectKey{Name: obj.GetName()}, existingCrd) + if apierrors.IsNotFound(err) { + if err = u.client.Create(ctx, &obj); err != nil { + return nil, errors.Wrapf(err, "failed to create CRD %s", obj.GetName()) + } + } else if err != nil { + return nil, errors.Wrapf(err, "failed to fetch state of %s", obj.GetName()) + } else { + obj.SetResourceVersion(existingCrd.GetResourceVersion()) + if err = u.client.Update(ctx, &obj); err != nil { + return nil, errors.Wrapf(err, "failed to update CRD %s", obj.GetName()) + } + } + } + + return crds, err +} + +func findCRDDirs(chartDir string) ([]string, error) { + dirs := make([]string, 0) + err := filepath.Walk(chartDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + if strings.HasSuffix(path, "crds") { + dirs = append(dirs, path) + } + return nil + } + return nil + }) + return dirs, err +} + +func parseChartCRDs(crds *[]unstructured.Unstructured, crdDir string) error { + errOuter := filepath.Walk(crdDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + // Add to CRDs .. + b, err := os.ReadFile(path) + if err != nil { + return err + } + + if len(b) == 0 { + return nil + } + + reader := k8syaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(b))) + doc, err := reader.Read() + if err != nil { + return err + } + + crd := unstructured.Unstructured{} + + dec := deser.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + + _, gvk, err := dec.Decode(doc, nil, &crd) + if err != nil { + return nil + } + + if gvk.Kind != "CustomResourceDefinition" { + return nil + } + + *crds = append(*crds, crd) + + return nil + }) + + return errOuter +} diff --git a/pkg/helmutil/crds_test.go b/pkg/helmutil/crds_test.go new file mode 100644 index 0000000..d9c4f61 --- /dev/null +++ b/pkg/helmutil/crds_test.go @@ -0,0 +1,71 @@ +package helmutil_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/k8ssandra/k8ssandra-client/pkg/helmutil" + "github.com/stretchr/testify/require" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func TestUpgradingCRDs(t *testing.T) { + require := require.New(t) + chartNames := []string{"cass-operator"} + for _, chartName := range chartNames { + namespace := env.CreateNamespace(t) + kubeClient := env.Client(namespace) + + // creating new upgrader + u, err := helmutil.NewUpgrader(kubeClient, helmutil.K8ssandraRepoName, helmutil.StableK8ssandraRepoURL, chartName) + require.NoError(err) + + crds, err := u.Upgrade(context.TODO(), "0.42.0") + require.NoError(err) + + testOptions := envtest.CRDInstallOptions{ + PollInterval: 100 * time.Millisecond, + MaxTime: 10 * time.Second, + } + + cassDCCRD := &apiextensions.CustomResourceDefinition{} + objs := []*apiextensions.CustomResourceDefinition{} + for _, crd := range crds { + if crd.GetName() == "cassandradatacenters.cassandra.datastax.com" { + err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), cassDCCRD) + require.NoError(err) + } + objs = append(objs, cassDCCRD) + } + + require.NotEmpty(objs) + require.NotEmpty(cassDCCRD.GetName()) + require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) + require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) + ver := cassDCCRD.GetResourceVersion() + + descRunsAsCassandra := cassDCCRD.Spec.Versions[0].DeepCopy().Schema.OpenAPIV3Schema.Properties["spec"].Properties["dockerImageRunsAsCassandra"].Description + require.False(strings.HasPrefix(descRunsAsCassandra, "DEPRECATED")) + + // Upgrading to 0.46.1 + _, err = u.Upgrade(context.TODO(), "0.46.1") + require.NoError(err) + + require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) + require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) + + require.Eventually(func() bool { + require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) + newver := cassDCCRD.GetResourceVersion() + return newver != ver + }, time.Minute*1, time.Second*5) + + descRunsAsCassandra = cassDCCRD.Spec.Versions[0].DeepCopy().Schema.OpenAPIV3Schema.Properties["spec"].Properties["dockerImageRunsAsCassandra"].Description + require.True(strings.HasPrefix(descRunsAsCassandra, "DEPRECATED")) + } +} diff --git a/pkg/helmutil/fetch.go b/pkg/helmutil/fetch.go index 3c57114..2beab78 100644 --- a/pkg/helmutil/fetch.go +++ b/pkg/helmutil/fetch.go @@ -77,9 +77,6 @@ func DownloadChartRelease(repoName, repoURL, chartName, targetVersion string) (s return "", err } - // TODO We can't do removeAll here.. - // defer os.RemoveAll(dir) - // _ is ProvenanceVerify (TODO we might want to verify the release) saved, _, err := c.DownloadTo(url, targetVersion, dir) if err != nil { @@ -89,16 +86,18 @@ func DownloadChartRelease(repoName, repoURL, chartName, targetVersion string) (s return saved, nil } -func ExtractChartRelease(saved, targetVersion string) (string, error) { - // TODO We need saved for the install process, clip from here to another function.. - +func ExtractChartRelease(saved, chartName, targetVersion string) (string, error) { // Extract the files - subDir := filepath.Join("helm", targetVersion) + subDir := filepath.Join(chartName, targetVersion) extractDir, err := util.GetCacheDir(subDir) if err != nil { return "", err } + if _, err := util.CreateIfNotExistsDir(extractDir); err != nil { + return "", err + } + // extractDir is a target directory err = chartutil.ExpandFile(extractDir, saved) if err != nil { @@ -108,6 +107,16 @@ func ExtractChartRelease(saved, targetVersion string) (string, error) { return extractDir, nil } +func GetChartTargetDir(chartName, targetVersion string) (string, error) { + subDir := filepath.Join(chartName, targetVersion) + extractDir, err := util.GetCacheDir(subDir) + if err != nil { + return "", err + } + + return extractDir, err +} + func Release(cfg *action.Configuration, releaseName string) (*release.Release, error) { getAction := action.NewGet(cfg) return getAction.Run(releaseName) diff --git a/pkg/helmutil/suite_test.go b/pkg/helmutil/suite_test.go new file mode 100644 index 0000000..388e42d --- /dev/null +++ b/pkg/helmutil/suite_test.go @@ -0,0 +1,16 @@ +package helmutil_test + +import ( + "os" + "testing" + + "github.com/k8ssandra/k8ssandra-client/internal/envtest" +) + +var ( + env *envtest.Environment +) + +func TestMain(m *testing.M) { + os.Exit(envtest.Run(m, func(e *envtest.Environment) { env = e })) +} diff --git a/pkg/util/fs.go b/pkg/util/fs.go index 15b6274..6e75da4 100644 --- a/pkg/util/fs.go +++ b/pkg/util/fs.go @@ -9,7 +9,7 @@ const ( dirSuffix = "k8ssandra" ) -// GetCacheDir returns the caching directory for k8ssandra and creates it if it does not exists +// GetCacheDir returns the caching directory for module func GetCacheDir(module string) (string, error) { userCacheDir, err := os.UserCacheDir() if err != nil { @@ -17,7 +17,7 @@ func GetCacheDir(module string) (string, error) { } targetDir := filepath.Join(userCacheDir, dirSuffix, module) - return createIfNotExistsDir(targetDir) + return targetDir, nil } // GetConfigDir returns the config directory for k8ssandra and creates it if it does not exists @@ -28,10 +28,10 @@ func GetConfigDir(module string) (string, error) { } targetDir := filepath.Join(userConfigDir, dirSuffix, module) - return createIfNotExistsDir(targetDir) + return CreateIfNotExistsDir(targetDir) } -func createIfNotExistsDir(targetDir string) (string, error) { +func CreateIfNotExistsDir(targetDir string) (string, error) { if _, err := os.Stat(targetDir); os.IsNotExist(err) { if err := os.MkdirAll(targetDir, 0755); err != nil { return "", err