From 738b978c1a9660215b893c5efda91d742e52c404 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 24 Jan 2024 21:09:17 +0200 Subject: [PATCH 1/3] Add modified version of the CRD upgrade tool for Helm installations --- cmd/kubectl-k8ssandra/helm/crds.go | 122 +++++++++++++++++++++++++++ cmd/kubectl-k8ssandra/helm/helm.go | 36 ++++++++ internal/envtest/envtest.go | 5 ++ pkg/helmutil/crds.go | 129 +++++++++++++++++++++++++++++ pkg/helmutil/crds_test.go | 94 +++++++++++++++++++++ pkg/helmutil/suite_test.go | 16 ++++ 6 files changed, 402 insertions(+) create mode 100644 cmd/kubectl-k8ssandra/helm/crds.go create mode 100644 cmd/kubectl-k8ssandra/helm/helm.go create mode 100644 pkg/helmutil/crds.go create mode 100644 pkg/helmutil/crds_test.go create mode 100644 pkg/helmutil/suite_test.go diff --git a/cmd/kubectl-k8ssandra/helm/crds.go b/cmd/kubectl-k8ssandra/helm/crds.go new file mode 100644 index 0000000..cf23aa5 --- /dev/null +++ b/cmd/kubectl-k8ssandra/helm/crds.go @@ -0,0 +1,122 @@ +package helm + +import ( + "context" + "fmt" + + "github.com/k8ssandra/k8ssandra-client/pkg/helmutil" + "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +var ( + upgraderExample = ` + # update CRDs in the namespace to targetVersion + %[1]s crds --chartName --targetVersion [] + + # update CRDs in the namespace to targetVersion with non-default chartRepo (helm.k8ssandra.io) + %[1]s crds --chartName --targetVersion --chartRepo [] + ` + errNotEnoughParameters = fmt.Errorf("not enough parameters, requires chartName and targetVersion") +) + +type options struct { + configFlags *genericclioptions.ConfigFlags + genericclioptions.IOStreams + namespace string + chartName string + targetVersion string + chartRepo string + repoURL string +} + +func newOptions(streams genericclioptions.IOStreams) *options { + return &options{ + configFlags: genericclioptions.NewConfigFlags(true), + IOStreams: streams, + } +} + +// NewCmd provides a cobra command wrapping cqlShOptions +func NewUpgradeCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := newOptions(streams) + + cmd := &cobra.Command{ + Use: "upgrade [flags]", + Short: "upgrade k8ssandra CRDs to target release version", + Example: fmt.Sprintf(upgraderExample, "kubectl k8ssandra helm crds"), + SilenceUsage: true, + RunE: func(c *cobra.Command, args []string) error { + if err := o.Complete(c, args); err != nil { + return err + } + if err := o.Validate(); err != nil { + return err + } + if err := o.Run(); err != nil { + return err + } + + return nil + }, + } + + fl := cmd.Flags() + fl.StringVar(&o.chartName, "chartName", "", "chartName to upgrade") + fl.StringVar(&o.targetVersion, "targetVersion", "", "targetVersion to upgrade to") + fl.StringVar(&o.chartRepo, "chartRepo", "", "optional chart repository name to override the default (k8ssandra)") + fl.StringVar(&o.repoURL, "repoURL", "", "optional chart repository url to override the default (helm.k8ssandra.io)") + o.configFlags.AddFlags(fl) + + return cmd +} + +// Complete parses the arguments and necessary flags to options +func (c *options) Complete(cmd *cobra.Command, args []string) error { + var err error + if len(args) < 2 { + return errNotEnoughParameters + } + + if c.repoURL == "" { + c.repoURL = helmutil.StableK8ssandraRepoURL + } + + if c.chartRepo == "" { + c.chartRepo = helmutil.K8ssandraRepoName + } + + c.targetVersion = args[0] + c.namespace, _, err = c.configFlags.ToRawKubeConfigLoader().Namespace() + return err +} + +// Validate ensures that all required arguments and flag values are provided +func (c *options) Validate() error { + // TODO Validate that the targetVersion is valid + return nil +} + +// Run removes the finalizers for a release X in the given namespace +func (c *options) Run() error { + restConfig, err := c.configFlags.ToRESTConfig() + if err != nil { + return err + } + + kubeClient, err := kubernetes.GetClientInNamespace(restConfig, c.namespace) + if err != nil { + return err + } + + ctx := context.Background() + + upgrader, err := helmutil.NewUpgrader(kubeClient, c.chartRepo, c.repoURL, c.chartName) + if err != nil { + return err + } + + _, err = upgrader.Upgrade(ctx, c.targetVersion) + return err +} diff --git a/cmd/kubectl-k8ssandra/helm/helm.go b/cmd/kubectl-k8ssandra/helm/helm.go new file mode 100644 index 0000000..8d958a3 --- /dev/null +++ b/cmd/kubectl-k8ssandra/helm/helm.go @@ -0,0 +1,36 @@ +package helm + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +type ClientOptions struct { + configFlags *genericclioptions.ConfigFlags + genericclioptions.IOStreams +} + +// NewClientOptions provides an instance of NamespaceOptions with default values +func NewHelmOptions(streams genericclioptions.IOStreams) *ClientOptions { + return &ClientOptions{ + configFlags: genericclioptions.NewConfigFlags(true), + IOStreams: streams, + } +} + +// NewCmd provides a cobra command wrapping NamespaceOptions +func NewHelmCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := NewHelmOptions(streams) + + cmd := &cobra.Command{ + Use: "k8ssandra [subcommand] [flags]", + } + + // Add subcommands + cmd.AddCommand(NewUpgradeCmd(streams)) + + // cmd.Flags().BoolVar(&o.listNamespaces, "list", o.listNamespaces, "if true, print the list of all namespaces in the current KUBECONFIG") + o.configFlags.AddFlags(cmd.Flags()) + + return cmd +} diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index ce7d6ba..e1d9033 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -12,6 +12,7 @@ import ( k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" + "k8s.io/client-go/rest" "k8s.io/kubectl/pkg/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -101,3 +102,7 @@ func (e *Environment) CreateNamespace(t *testing.T) string { return namespace } + +func (e *Environment) RestConfig() *rest.Config { + return e.env.Config +} diff --git a/pkg/helmutil/crds.go b/pkg/helmutil/crds.go new file mode 100644 index 0000000..371de7f --- /dev/null +++ b/pkg/helmutil/crds.go @@ -0,0 +1,129 @@ +package helmutil + +import ( + "bufio" + "bytes" + "context" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Upgrader is a utility to update the CRDs in a helm chart's pre-upgrade hook +type Upgrader struct { + client client.Client + repoName string + repoURL string + chartName string +} + +// NewUpgrader returns a new Upgrader client +func NewUpgrader(c client.Client, repoName, repoURL, chartName string) (*Upgrader, error) { + return &Upgrader{ + client: c, + repoName: repoName, + repoURL: repoURL, + chartName: chartName, + }, nil +} + +// Upgrade installs the missing CRDs or updates them if they exists already +func (u *Upgrader) Upgrade(ctx context.Context, targetVersion string) ([]unstructured.Unstructured, error) { + extractDir, err := DownloadChartRelease(u.repoName, u.repoURL, u.chartName, targetVersion) + if err != nil { + return nil, err + } + + // reaper and medusa subdirs have the required yaml files + chartPath := filepath.Join(extractDir, u.repoName) + defer os.RemoveAll(chartPath) + + crds := make([]unstructured.Unstructured, 0) + + // For each dir under the charts subdir, check the "crds/" + paths, _ := findCRDDirs(chartPath) + + for _, path := range paths { + err = parseChartCRDs(&crds, path) + if err != nil { + return nil, err + } + } + + var res []client.Object + for _, obj := range crds { + res = append(res, &obj) + } + + for _, obj := range res { + if err := u.client.Create(context.TODO(), obj); err != nil { + if apierrors.IsAlreadyExists(err) { + if err := u.client.Update(context.TODO(), obj); err != nil { + return nil, err + } + } + } + } + + return crds, err +} + +func findCRDDirs(chartDir string) ([]string, error) { + dirs := make([]string, 0) + err := filepath.Walk(chartDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + if strings.HasSuffix(path, "crds") { + dirs = append(dirs, path) + } + return nil + } + return nil + }) + return dirs, err +} + +func parseChartCRDs(crds *[]unstructured.Unstructured, crdDir string) error { + errOuter := filepath.Walk(crdDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + // Add to CRDs .. + b, err := os.ReadFile(path) + if err != nil { + return err + } + + reader := k8syaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(b))) + doc, err := reader.Read() + if err != nil { + return err + } + + crd := unstructured.Unstructured{} + + if err = yaml.Unmarshal(doc, &crd); err != nil { + return err + } + + *crds = append(*crds, crd) + return nil + }) + + return errOuter +} diff --git a/pkg/helmutil/crds_test.go b/pkg/helmutil/crds_test.go new file mode 100644 index 0000000..17105a8 --- /dev/null +++ b/pkg/helmutil/crds_test.go @@ -0,0 +1,94 @@ +package helmutil_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/k8ssandra/k8ssandra-client/pkg/helmutil" + "github.com/stretchr/testify/require" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + // +kubebuilder:scaffold:imports +) + +func TestUpgradingCRDs(t *testing.T) { + require := require.New(t) + chartNames := []string{"k8ssandra"} + for _, chartName := range chartNames { + t.Run(fmt.Sprintf("CRD upgrade for chart name %s", chartName), func(t *testing.T) { + namespace := env.CreateNamespace(t) + kubeClient := env.Client(namespace) + + // creating new upgrader + u, err := helmutil.NewUpgrader(kubeClient, helmutil.K8ssandraRepoName, helmutil.StableK8ssandraRepoURL, chartName) + require.NoError(err) + + // Upgrading / installing 1.0.0 + var crds []unstructured.Unstructured + require.Eventually(func() bool { + _, err := u.Upgrade(context.TODO(), "1.0.0") + return err == nil + }, time.Minute*1, time.Second*5) + + testOptions := envtest.CRDInstallOptions{ + PollInterval: 100 * time.Millisecond, + MaxTime: 10 * time.Second, + } + + unstructuredCRD := &unstructured.Unstructured{} + cassDCCRD := &apiextensions.CustomResourceDefinition{} + objs := []*apiextensions.CustomResourceDefinition{} + for _, crd := range crds { + if crd.GetName() == "cassandradatacenters.cassandra.datastax.com" { + unstructuredCRD = crd.DeepCopy() + err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), cassDCCRD) + require.NoError(err) + } + objs = append(objs, cassDCCRD) + } + + require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) + require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) + ver := cassDCCRD.GetResourceVersion() + + _, found, err := unstructured.NestedFieldNoCopy(unstructuredCRD.Object, "spec", "validation", "openAPIV3Schema", "properties", "spec", "properties", "configSecret") + require.NoError(err) + require.False(found) + + // Upgrading to 1.5.1 + crds, err = u.Upgrade(context.TODO(), "1.5.1") + require.NoError(err) + + objs = []*apiextensions.CustomResourceDefinition{} + for _, crd := range crds { + if crd.GetName() == "cassandradatacenters.cassandra.datastax.com" { + unstructuredCRD = crd.DeepCopy() + err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), cassDCCRD) + require.NoError(err) + objs = append(objs, cassDCCRD) + } + } + + require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) + require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) + + require.Eventually(func() bool { + newver := cassDCCRD.GetResourceVersion() + return newver == ver + }, time.Minute*1, time.Second*5) + + versionsSlice, found, err := unstructured.NestedSlice(unstructuredCRD.Object, "spec", "versions") + require.NoError(err) + require.True(found) + + _, found, err = unstructured.NestedFieldNoCopy(versionsSlice[0].(map[string]interface{}), "schema", "openAPIV3Schema", "properties", "spec", "properties", "configSecret") + require.NoError(err) + require.True(found) + }) + } +} diff --git a/pkg/helmutil/suite_test.go b/pkg/helmutil/suite_test.go new file mode 100644 index 0000000..388e42d --- /dev/null +++ b/pkg/helmutil/suite_test.go @@ -0,0 +1,16 @@ +package helmutil_test + +import ( + "os" + "testing" + + "github.com/k8ssandra/k8ssandra-client/internal/envtest" +) + +var ( + env *envtest.Environment +) + +func TestMain(m *testing.M) { + os.Exit(envtest.Run(m, func(e *envtest.Environment) { env = e })) +} From 8838f76fe4aec0b299c3b2a5c91d757ed05f6fe8 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Thu, 25 Jan 2024 17:39:44 +0200 Subject: [PATCH 2/3] Simplify certain methods, modify test --- Makefile | 2 +- internal/envtest/envtest.go | 5 ++ pkg/helmutil/crds.go | 54 ++++++++++++------- pkg/helmutil/crds_test.go | 101 ++++++++++++++---------------------- 4 files changed, 81 insertions(+), 81 deletions(-) diff --git a/Makefile b/Makefile index f5889f5..21e4c7c 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.27.x +ENVTEST_K8S_VERSION = 1.28.x .PHONY: all all: build diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index e1d9033..e04bf5d 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -12,6 +12,7 @@ import ( k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/rest" "k8s.io/kubectl/pkg/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -77,6 +78,10 @@ func (e *Environment) start() { panic(err) } + if err := apiextensions.AddToScheme(scheme.Scheme); err != nil { + panic(err) + } + //+kubebuilder:scaffold:scheme k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/pkg/helmutil/crds.go b/pkg/helmutil/crds.go index 371de7f..c64dc24 100644 --- a/pkg/helmutil/crds.go +++ b/pkg/helmutil/crds.go @@ -8,10 +8,10 @@ import ( "path/filepath" "strings" - "gopkg.in/yaml.v3" - + "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + deser "k8s.io/apimachinery/pkg/runtime/serializer/yaml" k8syaml "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,14 +37,18 @@ func NewUpgrader(c client.Client, repoName, repoURL, chartName string) (*Upgrade // Upgrade installs the missing CRDs or updates them if they exists already func (u *Upgrader) Upgrade(ctx context.Context, targetVersion string) ([]unstructured.Unstructured, error) { - extractDir, err := DownloadChartRelease(u.repoName, u.repoURL, u.chartName, targetVersion) + downloadDir, err := DownloadChartRelease(u.repoName, u.repoURL, u.chartName, targetVersion) + if err != nil { + return nil, err + } + + extractDir, err := ExtractChartRelease(downloadDir, targetVersion) if err != nil { return nil, err } - // reaper and medusa subdirs have the required yaml files - chartPath := filepath.Join(extractDir, u.repoName) - defer os.RemoveAll(chartPath) + chartPath := filepath.Join(extractDir, u.chartName) + defer os.RemoveAll(downloadDir) crds := make([]unstructured.Unstructured, 0) @@ -58,17 +62,19 @@ func (u *Upgrader) Upgrade(ctx context.Context, targetVersion string) ([]unstruc } } - var res []client.Object for _, obj := range crds { - res = append(res, &obj) - } - - for _, obj := range res { - if err := u.client.Create(context.TODO(), obj); err != nil { - if apierrors.IsAlreadyExists(err) { - if err := u.client.Update(context.TODO(), obj); err != nil { - return nil, err - } + existingCrd := obj.DeepCopy() + err = u.client.Get(context.TODO(), client.ObjectKey{Name: obj.GetName()}, existingCrd) + if apierrors.IsNotFound(err) { + if err = u.client.Create(context.TODO(), &obj); err != nil { + return nil, errors.Wrapf(err, "failed to create CRD %s", obj.GetName()) + } + } else if err != nil { + return nil, errors.Wrapf(err, "failed to fetch state of %s", obj.GetName()) + } else { + obj.SetResourceVersion(existingCrd.GetResourceVersion()) + if err = u.client.Update(context.TODO(), &obj); err != nil { + return nil, errors.Wrapf(err, "failed to update CRD %s", obj.GetName()) } } } @@ -109,6 +115,10 @@ func parseChartCRDs(crds *[]unstructured.Unstructured, crdDir string) error { return err } + if len(b) == 0 { + return nil + } + reader := k8syaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(b))) doc, err := reader.Read() if err != nil { @@ -117,11 +127,19 @@ func parseChartCRDs(crds *[]unstructured.Unstructured, crdDir string) error { crd := unstructured.Unstructured{} - if err = yaml.Unmarshal(doc, &crd); err != nil { - return err + dec := deser.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + + _, gvk, err := dec.Decode(doc, nil, &crd) + if err != nil { + return nil + } + + if gvk.Kind != "CustomResourceDefinition" { + return nil } *crds = append(*crds, crd) + return nil }) diff --git a/pkg/helmutil/crds_test.go b/pkg/helmutil/crds_test.go index 17105a8..d9c4f61 100644 --- a/pkg/helmutil/crds_test.go +++ b/pkg/helmutil/crds_test.go @@ -2,93 +2,70 @@ package helmutil_test import ( "context" - "fmt" + "strings" "testing" "time" "github.com/k8ssandra/k8ssandra-client/pkg/helmutil" "github.com/stretchr/testify/require" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - // +kubebuilder:scaffold:imports ) func TestUpgradingCRDs(t *testing.T) { require := require.New(t) - chartNames := []string{"k8ssandra"} + chartNames := []string{"cass-operator"} for _, chartName := range chartNames { - t.Run(fmt.Sprintf("CRD upgrade for chart name %s", chartName), func(t *testing.T) { - namespace := env.CreateNamespace(t) - kubeClient := env.Client(namespace) + namespace := env.CreateNamespace(t) + kubeClient := env.Client(namespace) - // creating new upgrader - u, err := helmutil.NewUpgrader(kubeClient, helmutil.K8ssandraRepoName, helmutil.StableK8ssandraRepoURL, chartName) - require.NoError(err) + // creating new upgrader + u, err := helmutil.NewUpgrader(kubeClient, helmutil.K8ssandraRepoName, helmutil.StableK8ssandraRepoURL, chartName) + require.NoError(err) - // Upgrading / installing 1.0.0 - var crds []unstructured.Unstructured - require.Eventually(func() bool { - _, err := u.Upgrade(context.TODO(), "1.0.0") - return err == nil - }, time.Minute*1, time.Second*5) + crds, err := u.Upgrade(context.TODO(), "0.42.0") + require.NoError(err) - testOptions := envtest.CRDInstallOptions{ - PollInterval: 100 * time.Millisecond, - MaxTime: 10 * time.Second, - } + testOptions := envtest.CRDInstallOptions{ + PollInterval: 100 * time.Millisecond, + MaxTime: 10 * time.Second, + } - unstructuredCRD := &unstructured.Unstructured{} - cassDCCRD := &apiextensions.CustomResourceDefinition{} - objs := []*apiextensions.CustomResourceDefinition{} - for _, crd := range crds { - if crd.GetName() == "cassandradatacenters.cassandra.datastax.com" { - unstructuredCRD = crd.DeepCopy() - err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), cassDCCRD) - require.NoError(err) - } - objs = append(objs, cassDCCRD) + cassDCCRD := &apiextensions.CustomResourceDefinition{} + objs := []*apiextensions.CustomResourceDefinition{} + for _, crd := range crds { + if crd.GetName() == "cassandradatacenters.cassandra.datastax.com" { + err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), cassDCCRD) + require.NoError(err) } + objs = append(objs, cassDCCRD) + } - require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) - require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) - ver := cassDCCRD.GetResourceVersion() + require.NotEmpty(objs) + require.NotEmpty(cassDCCRD.GetName()) + require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) + require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) + ver := cassDCCRD.GetResourceVersion() - _, found, err := unstructured.NestedFieldNoCopy(unstructuredCRD.Object, "spec", "validation", "openAPIV3Schema", "properties", "spec", "properties", "configSecret") - require.NoError(err) - require.False(found) + descRunsAsCassandra := cassDCCRD.Spec.Versions[0].DeepCopy().Schema.OpenAPIV3Schema.Properties["spec"].Properties["dockerImageRunsAsCassandra"].Description + require.False(strings.HasPrefix(descRunsAsCassandra, "DEPRECATED")) - // Upgrading to 1.5.1 - crds, err = u.Upgrade(context.TODO(), "1.5.1") - require.NoError(err) + // Upgrading to 0.46.1 + _, err = u.Upgrade(context.TODO(), "0.46.1") + require.NoError(err) - objs = []*apiextensions.CustomResourceDefinition{} - for _, crd := range crds { - if crd.GetName() == "cassandradatacenters.cassandra.datastax.com" { - unstructuredCRD = crd.DeepCopy() - err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), cassDCCRD) - require.NoError(err) - objs = append(objs, cassDCCRD) - } - } + require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) + require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) - require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions)) + require.Eventually(func() bool { require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: cassDCCRD.GetName()}, cassDCCRD)) + newver := cassDCCRD.GetResourceVersion() + return newver != ver + }, time.Minute*1, time.Second*5) - require.Eventually(func() bool { - newver := cassDCCRD.GetResourceVersion() - return newver == ver - }, time.Minute*1, time.Second*5) - - versionsSlice, found, err := unstructured.NestedSlice(unstructuredCRD.Object, "spec", "versions") - require.NoError(err) - require.True(found) - - _, found, err = unstructured.NestedFieldNoCopy(versionsSlice[0].(map[string]interface{}), "schema", "openAPIV3Schema", "properties", "spec", "properties", "configSecret") - require.NoError(err) - require.True(found) - }) + descRunsAsCassandra = cassDCCRD.Spec.Versions[0].DeepCopy().Schema.OpenAPIV3Schema.Properties["spec"].Properties["dockerImageRunsAsCassandra"].Description + require.True(strings.HasPrefix(descRunsAsCassandra, "DEPRECATED")) } } From 068dccb2110d3ed763ca03e1c98763ce9b6e7210 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 26 Jan 2024 17:57:30 +0200 Subject: [PATCH 3/3] Fix download caching --- pkg/helmutil/crds.go | 27 +++++++++++++++++---------- pkg/helmutil/fetch.go | 23 ++++++++++++++++------- pkg/util/fs.go | 8 ++++---- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/pkg/helmutil/crds.go b/pkg/helmutil/crds.go index c64dc24..516eeb9 100644 --- a/pkg/helmutil/crds.go +++ b/pkg/helmutil/crds.go @@ -37,23 +37,30 @@ func NewUpgrader(c client.Client, repoName, repoURL, chartName string) (*Upgrade // Upgrade installs the missing CRDs or updates them if they exists already func (u *Upgrader) Upgrade(ctx context.Context, targetVersion string) ([]unstructured.Unstructured, error) { - downloadDir, err := DownloadChartRelease(u.repoName, u.repoURL, u.chartName, targetVersion) + chartDir, err := GetChartTargetDir(u.chartName, targetVersion) if err != nil { return nil, err } - extractDir, err := ExtractChartRelease(downloadDir, targetVersion) - if err != nil { - return nil, err + if _, err := os.Stat(chartDir); os.IsNotExist(err) { + downloadDir, err := DownloadChartRelease(u.repoName, u.repoURL, u.chartName, targetVersion) + if err != nil { + return nil, err + } + + extractDir, err := ExtractChartRelease(downloadDir, u.chartName, targetVersion) + if err != nil { + return nil, err + } + chartDir = extractDir } - chartPath := filepath.Join(extractDir, u.chartName) - defer os.RemoveAll(downloadDir) + // defer os.RemoveAll(downloadDir) crds := make([]unstructured.Unstructured, 0) // For each dir under the charts subdir, check the "crds/" - paths, _ := findCRDDirs(chartPath) + paths, _ := findCRDDirs(chartDir) for _, path := range paths { err = parseChartCRDs(&crds, path) @@ -64,16 +71,16 @@ func (u *Upgrader) Upgrade(ctx context.Context, targetVersion string) ([]unstruc for _, obj := range crds { existingCrd := obj.DeepCopy() - err = u.client.Get(context.TODO(), client.ObjectKey{Name: obj.GetName()}, existingCrd) + err = u.client.Get(ctx, client.ObjectKey{Name: obj.GetName()}, existingCrd) if apierrors.IsNotFound(err) { - if err = u.client.Create(context.TODO(), &obj); err != nil { + if err = u.client.Create(ctx, &obj); err != nil { return nil, errors.Wrapf(err, "failed to create CRD %s", obj.GetName()) } } else if err != nil { return nil, errors.Wrapf(err, "failed to fetch state of %s", obj.GetName()) } else { obj.SetResourceVersion(existingCrd.GetResourceVersion()) - if err = u.client.Update(context.TODO(), &obj); err != nil { + if err = u.client.Update(ctx, &obj); err != nil { return nil, errors.Wrapf(err, "failed to update CRD %s", obj.GetName()) } } diff --git a/pkg/helmutil/fetch.go b/pkg/helmutil/fetch.go index 3c57114..2beab78 100644 --- a/pkg/helmutil/fetch.go +++ b/pkg/helmutil/fetch.go @@ -77,9 +77,6 @@ func DownloadChartRelease(repoName, repoURL, chartName, targetVersion string) (s return "", err } - // TODO We can't do removeAll here.. - // defer os.RemoveAll(dir) - // _ is ProvenanceVerify (TODO we might want to verify the release) saved, _, err := c.DownloadTo(url, targetVersion, dir) if err != nil { @@ -89,16 +86,18 @@ func DownloadChartRelease(repoName, repoURL, chartName, targetVersion string) (s return saved, nil } -func ExtractChartRelease(saved, targetVersion string) (string, error) { - // TODO We need saved for the install process, clip from here to another function.. - +func ExtractChartRelease(saved, chartName, targetVersion string) (string, error) { // Extract the files - subDir := filepath.Join("helm", targetVersion) + subDir := filepath.Join(chartName, targetVersion) extractDir, err := util.GetCacheDir(subDir) if err != nil { return "", err } + if _, err := util.CreateIfNotExistsDir(extractDir); err != nil { + return "", err + } + // extractDir is a target directory err = chartutil.ExpandFile(extractDir, saved) if err != nil { @@ -108,6 +107,16 @@ func ExtractChartRelease(saved, targetVersion string) (string, error) { return extractDir, nil } +func GetChartTargetDir(chartName, targetVersion string) (string, error) { + subDir := filepath.Join(chartName, targetVersion) + extractDir, err := util.GetCacheDir(subDir) + if err != nil { + return "", err + } + + return extractDir, err +} + func Release(cfg *action.Configuration, releaseName string) (*release.Release, error) { getAction := action.NewGet(cfg) return getAction.Run(releaseName) diff --git a/pkg/util/fs.go b/pkg/util/fs.go index 15b6274..6e75da4 100644 --- a/pkg/util/fs.go +++ b/pkg/util/fs.go @@ -9,7 +9,7 @@ const ( dirSuffix = "k8ssandra" ) -// GetCacheDir returns the caching directory for k8ssandra and creates it if it does not exists +// GetCacheDir returns the caching directory for module func GetCacheDir(module string) (string, error) { userCacheDir, err := os.UserCacheDir() if err != nil { @@ -17,7 +17,7 @@ func GetCacheDir(module string) (string, error) { } targetDir := filepath.Join(userCacheDir, dirSuffix, module) - return createIfNotExistsDir(targetDir) + return targetDir, nil } // GetConfigDir returns the config directory for k8ssandra and creates it if it does not exists @@ -28,10 +28,10 @@ func GetConfigDir(module string) (string, error) { } targetDir := filepath.Join(userConfigDir, dirSuffix, module) - return createIfNotExistsDir(targetDir) + return CreateIfNotExistsDir(targetDir) } -func createIfNotExistsDir(targetDir string) (string, error) { +func CreateIfNotExistsDir(targetDir string) (string, error) { if _, err := os.Stat(targetDir); os.IsNotExist(err) { if err := os.MkdirAll(targetDir, 0755); err != nil { return "", err