issue-467, use the latest clusters version for packaged provisioning …

…was implemented
instaclustr · Jul 13, 2023 · c061168 · c061168
1 parent b250393
commit c061168
Show file tree

Hide file tree

Showing 9 changed files with 101 additions and 38 deletions.
diff --git a/apis/clusters/v1beta1/opensearch_types.go b/apis/clusters/v1beta1/opensearch_types.go
@@ -48,7 +48,7 @@ type OpenSearchSpec struct {
 	NotificationsPlugin      bool                    `json:"notificationsPlugin,omitempty"`
 	AnomalyDetectionPlugin   bool                    `json:"anomalyDetectionPlugin,omitempty"`
 	LoadBalancer             bool                    `json:"loadBalancer,omitempty"`
-	ClusterManagerNodes      []*ClusterManagerNodes  `json:"clusterManagerNodes"`
+	ClusterManagerNodes      []*ClusterManagerNodes  `json:"clusterManagerNodes,omitempty"`
 	IndexManagementPlugin    bool                    `json:"indexManagementPlugin,omitempty"`
 	AlertingPlugin           bool                    `json:"alertingPlugin,omitempty"`
 	BundledUseOnly           bool                    `json:"bundleUseOnly,omitempty"`

diff --git a/config/crd/bases/clusters.instaclustr.com_opensearches.yaml b/config/crd/bases/clusters.instaclustr.com_opensearches.yaml
@@ -224,8 +224,6 @@ spec:
                 type: object
               version:
                 type: string
-            required:
-            - clusterManagerNodes
             type: object
           status:
             description: OpenSearchStatus defines the observed state of OpenSearch

diff --git a/config/samples/clusters_v1beta1_cadence.yaml b/config/samples/clusters_v1beta1_cadence.yaml
@@ -12,31 +12,31 @@ metadata:
   name: cadence-sample
 spec:
   name: "operatorCadence"
-  version: "0.24.0"
+  version: "1.0.0"
   standardProvisioning:
     - targetCassandra:
         dependencyCdcId: "9d43ac54-7317-4ce5-859a-e9d0443508a4"
         dependencyVpcType: "VPC_PEERED"
-  #  packagedProvisioning:
-  #    - bundledCassandraSpec:
-  #        nodeSize: "CAS-DEV-t4g.small-5"
-  #        network: "10.2.0.0/16"
-  #        replicationFactor: 3
-  #        nodesNumber: 3
-  #        privateIPBroadcastForDiscovery: false
-  #        passwordAndUserAuth: true
-  #      useAdvancedVisibility: true
-  #      bundledKafkaSpec:
-  #        nodeSize: "KFK-DEV-t4g.small-5"
-  #        nodesNumber: 3
-  #        network: "10.3.0.0/16"
-  #        replicationFactor: 3
-  #        partitionsNumber: 3
-  #      bundledOpenSearchSpec:
-  #        nodeSize: "SRH-DEV-t4g.small-5"
-  #        replicationFactor: 3
-  #        nodesPerReplicationFactor: 1
-  #        network: "10.4.0.0/16"
+#  packagedProvisioning:
+#    - bundledCassandraSpec:
+#        nodeSize: "CAS-DEV-t4g.small-5"
+#        network: "10.2.0.0/16"
+#        replicationFactor: 3
+#        nodesNumber: 3
+#        privateIPBroadcastForDiscovery: false
+#        passwordAndUserAuth: true
+#      useAdvancedVisibility: true
+#      bundledKafkaSpec:
+#        nodeSize: "KFK-DEV-t4g.small-5"
+#        nodesNumber: 3
+#        network: "10.3.0.0/16"
+#        replicationFactor: 3
+#        partitionsNumber: 3
+#      bundledOpenSearchSpec:
+#        nodeSize: "SRH-DEV-t4g.small-5"
+#        replicationFactor: 3
+#        nodesPerReplicationFactor: 3
+#        network: "10.4.0.0/16"
   #  twoFactorDelete:
   #  - email: "[email protected]"
   privateNetworkCluster: false

diff --git a/controllers/clusters/cadence_controller.go b/controllers/clusters/cadence_controller.go
@@ -527,7 +527,19 @@ func (r *CadenceReconciler) preparePackagedSolution(
 	}
 
 	if len(cassandraList.Items) == 0 {
-		cassandraSpec, err := r.newCassandraSpec(cluster)
+		appVersions, err := r.API.ListAppVersions(models.CassandraAppKind)
+		if err != nil {
+			return false, fmt.Errorf("cannot list versions for kind: %v, err: %w",
+				models.CassandraAppKind, err)
+		}
+
+		cassandraVersions := getSortedAppVersions(appVersions, models.CassandraAppType)
+		if len(cassandraVersions) == 0 {
+			return false, fmt.Errorf("there are no versions for %v kind",
+				models.CassandraAppKind)
+		}
+
+		cassandraSpec, err := r.newCassandraSpec(cluster, cassandraVersions[len(cassandraVersions)-1].String())
 		if err != nil {
 			return false, err
 		}
@@ -553,7 +565,19 @@ func (r *CadenceReconciler) preparePackagedSolution(
 			return false, err
 		}
 		if len(kafkaList.Items) == 0 {
-			kafkaSpec, err := r.newKafkaSpec(cluster)
+			appVersions, err := r.API.ListAppVersions(models.KafkaAppKind)
+			if err != nil {
+				return false, fmt.Errorf("cannot list versions for kind: %v, err: %w",
+					models.KafkaAppKind, err)
+			}
+
+			kafkaVersions := getSortedAppVersions(appVersions, models.KafkaAppType)
+			if len(kafkaVersions) == 0 {
+				return false, fmt.Errorf("there are no versions for %v kind",
+					models.KafkaAppType)
+			}
+
+			kafkaSpec, err := r.newKafkaSpec(cluster, kafkaVersions[len(kafkaVersions)-1].String())
 			if err != nil {
 				return false, err
 			}
@@ -578,7 +602,20 @@ func (r *CadenceReconciler) preparePackagedSolution(
 			return false, err
 		}
 		if len(osList.Items) == 0 {
-			osSpec, err := r.newOpenSearchSpec(cluster)
+			appVersions, err := r.API.ListAppVersions(models.OpenSearchAppKind)
+			if err != nil {
+				return false, fmt.Errorf("cannot list versions for kind: %v, err: %w",
+					models.OpenSearchAppKind, err)
+			}
+
+			openSearchVersions := getSortedAppVersions(appVersions, models.OpenSearchAppType)
+			if len(openSearchVersions) == 0 {
+				return false, fmt.Errorf("there are no versions for %v kind",
+					models.OpenSearchAppType)
+			}
+
+			// For OpenSearch we cannot use the latest version because is not supported by Cadence. So we use the oldest one.
+			osSpec, err := r.newOpenSearchSpec(cluster, openSearchVersions[0].String())
 			if err != nil {
 				return false, err
 			}
@@ -615,7 +652,7 @@ func (r *CadenceReconciler) preparePackagedSolution(
 	return false, nil
 }
 
-func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence) (*v1beta1.Cassandra, error) {
+func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence, latestCassandraVersion string) (*v1beta1.Cassandra, error) {
 	typeMeta := v1.TypeMeta{
 		Kind:       models.CassandraKind,
 		APIVersion: models.ClustersV1beta1APIVersion,
@@ -691,7 +728,7 @@ func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence) (*v1beta1
 	spec := v1beta1.CassandraSpec{
 		Cluster: v1beta1.Cluster{
 			Name:                  models.CassandraChildPrefix + cadence.Name,
-			Version:               models.CassandraV3_11_13,
+			Version:               latestCassandraVersion,
 			SLATier:               slaTier,
 			PrivateNetworkCluster: privateClusterNetwork,
 			TwoFactorDelete:       twoFactorDelete,
@@ -884,7 +921,7 @@ func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) schedule
 	}
 }
 
-func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence) (*v1beta1.Kafka, error) {
+func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence, latestKafkaVersion string) (*v1beta1.Kafka, error) {
 	typeMeta := v1.TypeMeta{
 		Kind:       models.KafkaKind,
 		APIVersion: models.ClustersV1beta1APIVersion,
@@ -950,7 +987,7 @@ func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence) (*v1beta1.Kaf
 	spec := v1beta1.KafkaSpec{
 		Cluster: v1beta1.Cluster{
 			Name:                  models.KafkaChildPrefix + cadence.Name,
-			Version:               models.KafkaV3_1_2,
+			Version:               latestKafkaVersion,
 			SLATier:               slaTier,
 			PrivateNetworkCluster: privateClusterNetwork,
 			TwoFactorDelete:       kafkaTFD,
@@ -971,7 +1008,7 @@ func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence) (*v1beta1.Kaf
 	}, nil
 }
 
-func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence) (*v1beta1.OpenSearch, error) {
+func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence, oldestOpenSearchVersion string) (*v1beta1.OpenSearch, error) {
 	typeMeta := v1.TypeMeta{
 		Kind:       models.OpenSearchKind,
 		APIVersion: models.ClustersV1beta1APIVersion,
@@ -1039,12 +1076,18 @@ func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence) (*v1beta
 	spec := v1beta1.OpenSearchSpec{
 		Cluster: v1beta1.Cluster{
 			Name:                  models.OpenSearchChildPrefix + cadence.Name,
-			Version:               models.OpenSearchV1_3_7,
+			Version:               oldestOpenSearchVersion,
 			SLATier:               slaTier,
 			PrivateNetworkCluster: privateClusterNetwork,
 			TwoFactorDelete:       twoFactorDelete,
 			PCICompliance:         pciCompliance,
 		},
+		ClusterManagerNodes: []*v1beta1.ClusterManagerNodes{
+			{
+				NodeSize:         osNodeSize,
+				DedicatedManager: false,
+			},
+		},
 		DataCentres: osDataCentres,
 	}
 

diff --git a/controllers/clusters/helpers.go b/controllers/clusters/helpers.go
@@ -17,6 +17,9 @@ limitations under the License.
 package clusters
 
 import (
+	"sort"
+
+	"github.com/hashicorp/go-version"
 	"k8s.io/utils/strings/slices"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -134,6 +137,24 @@ func isClusterActive(clusterID string, activeClusters []*models.ActiveClusters)
 	return false
 }
 
+func getSortedAppVersions(versions []*models.AppVersions, appType string) []*version.Version {
+	for _, apps := range versions {
+		if apps.Application == appType {
+			newVersions := make([]*version.Version, len(apps.Versions))
+			for i, versionStr := range apps.Versions {
+				v, _ := version.NewVersion(versionStr)
+				newVersions[i] = v
+			}
+
+			sort.Sort(version.Collection(newVersions))
+
+			return newVersions
+		}
+	}
+
+	return nil
+}
+
 var msgDeleteClusterWithTwoFactorDelete = "Please confirm cluster deletion via email or phone. " +
 	"If you have canceled a cluster deletion and want to put the cluster on deletion again, " +
 	"remove \"triggered\" from Instaclustr.com/clusterDeletion annotation."

diff --git a/go.mod b/go.mod
@@ -5,6 +5,7 @@ go 1.18
 require (
 	github.com/go-logr/logr v1.2.0
 	github.com/gorilla/mux v1.8.0
+	github.com/hashicorp/go-version v1.6.0
 	github.com/onsi/ginkgo/v2 v2.0.0
 	github.com/onsi/gomega v1.18.1
 	k8s.io/api v0.24.2

diff --git a/go.sum b/go.sum
@@ -277,6 +277,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

diff --git a/pkg/instaclustr/interfaces.go b/pkg/instaclustr/interfaces.go
@@ -94,4 +94,5 @@ type API interface {
 	DeleteEncryptionKey(encryptionKeyID string) error
 	CreateUser(userSpec any, clusterID, app string) error
 	DeleteUser(username, clusterID, app string) error
+	ListAppVersions(app string) ([]*models.AppVersions, error)
 }
diff --git a/pkg/models/operator.go b/pkg/models/operator.go
@@ -53,11 +53,8 @@ const (
 	OpenSearchChildPrefix = "opensearch-"
 	OpenSearchChildDCName = "opensearch-cadence-dc"
 
-	CassandraV3_11_13 = "3.11.13"
-	KafkaV3_1_2       = "3.1.2"
-	OpenSearchV1_3_7  = "1.3.7"
-	K8sAPIVersionV1   = "v1"
-	VPCPeered         = "VPC_PEERED"
+	K8sAPIVersionV1 = "v1"
+	VPCPeered       = "VPC_PEERED"
 
 	True  = "true"
 	False = "false"