From ac0fe2cf129e0e9035f12cae50eb5ec98c0a7f3b Mon Sep 17 00:00:00 2001 From: melserngawy Date: Mon, 11 Sep 2023 17:36:16 -0400 Subject: [PATCH] Update rollout lib Signed-off-by: melserngawy --- cluster/v1alpha1/helpers.go | 294 ++-- cluster/v1alpha1/helpers_test.go | 1858 ++++++++++++++++----- cluster/v1alpha1/zz_generated.deepcopy.go | 4 +- 3 files changed, 1619 insertions(+), 537 deletions(-) diff --git a/cluster/v1alpha1/helpers.go b/cluster/v1alpha1/helpers.go index d70865825..9d25e05d5 100644 --- a/cluster/v1alpha1/helpers.go +++ b/cluster/v1alpha1/helpers.go @@ -37,11 +37,10 @@ const ( Skip ) -// ClusterRolloutStatusFunc defines a function to return the rollout status for a managed cluster. -type ClusterRolloutStatusFunc func(clusterName string) ClusterRolloutStatus - // ClusterRolloutStatus holds the rollout status information for a cluster. type ClusterRolloutStatus struct { + // cluster name + ClusterName string // GroupKey represents the cluster group key (optional field). GroupKey clusterv1beta1.GroupKey // Status is the required field indicating the rollout status. @@ -53,50 +52,57 @@ type ClusterRolloutStatus struct { TimeOutTime *metav1.Time } -// RolloutResult contains the clusters to be rolled out and the clusters that have timed out. +// RolloutResult contains list of clusters that are timeOut, removed and required to rollOut type RolloutResult struct { - // ClustersToRollout is a map where the key is the cluster name and the value is the ClusterRolloutStatus. - ClustersToRollout map[string]ClusterRolloutStatus - // ClustersTimeOut is a map where the key is the cluster name and the value is the ClusterRolloutStatus. - ClustersTimeOut map[string]ClusterRolloutStatus + // ClustersToRollout is a slice of ClusterRolloutStatus that will be rolled out. + ClustersToRollout []ClusterRolloutStatus + // ClustersTimeOut is a slice of ClusterRolloutStatus that are timeout. + ClustersTimeOut []ClusterRolloutStatus + // ClustersRemoved is a slice of ClusterRolloutStatus that are removed. + ClustersRemoved []ClusterRolloutStatus } +// ClusterRolloutStatusFunc defines a function that return the rollout status for a given workload. +type ClusterRolloutStatusFunc[T any] func(clusterName string, workload T) (ClusterRolloutStatus, error) + +// The RolloutHandler required workload type (interface/struct) to be assigned to the generic type. +// The custom implementation of the ClusterRolloutStatusFunc is required to use the RolloutHandler. // +k8s:deepcopy-gen=false -type RolloutHandler struct { +type RolloutHandler[T any] struct { // placement decision tracker - pdTracker *clusterv1beta1.PlacementDecisionClustersTracker + pdTracker *clusterv1beta1.PlacementDecisionClustersTracker + statusFunc ClusterRolloutStatusFunc[T] } -func NewRolloutHandler(pdTracker *clusterv1beta1.PlacementDecisionClustersTracker) (*RolloutHandler, error) { +// NewRolloutHandler creates a new RolloutHandler with the give workload type. +func NewRolloutHandler[T any](pdTracker *clusterv1beta1.PlacementDecisionClustersTracker, statusFunc ClusterRolloutStatusFunc[T]) (*RolloutHandler[T], error) { if pdTracker == nil { return nil, fmt.Errorf("invalid placement decision tracker %v", pdTracker) } - return &RolloutHandler{pdTracker: pdTracker}, nil + return &RolloutHandler[T]{pdTracker: pdTracker, statusFunc: statusFunc}, nil } -// The input is a duck type RolloutStrategy and a ClusterRolloutStatusFunc to return the rollout status on each managed cluster. -// Return the strategy actual take effect and a list of clusters that need to rollout and that are timeout. +// The input are a RolloutStrategy and existingClusterRolloutStatus list. +// The existing ClusterRolloutStatus list should be created using the ClusterRolloutStatusFunc to determine the current workload rollout status. +// The existing ClusterRolloutStatus list should contain all the current workloads rollout status such as ToApply, Progressing, Succeeded, +// Failed, TimeOut and Skip in order to determine the added, removed, timeout clusters and next clusters to rollout. // -// ClustersToRollout: If mandatory decision groups are defined in strategy, will return the clusters to rollout in mandatory decision groups first. -// When all the mandatory decision groups rollout successfully, will return the rest of the clusters that need to rollout. -// -// ClustersTimeOut: If the cluster status is Progressing or Failed, and the status lasts longer than timeout defined in strategy, -// will list them RolloutResult.ClustersTimeOut with status TimeOut. -func (r *RolloutHandler) GetRolloutCluster(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +// Return the actual RolloutStrategy that take effect and a RolloutResult contain list of ClusterToRollout, ClustersTimeout and ClusterRemoved. +func (r *RolloutHandler[T]) GetRolloutCluster(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { switch rolloutStrategy.Type { case All: - return r.getRolloutAllClusters(rolloutStrategy, statusFunc) + return r.getRolloutAllClusters(rolloutStrategy, existingClusterStatus) case Progressive: - return r.getProgressiveClusters(rolloutStrategy, statusFunc) + return r.getProgressiveClusters(rolloutStrategy, existingClusterStatus) case ProgressivePerGroup: - return r.getProgressivePerGroupClusters(rolloutStrategy, statusFunc) + return r.getProgressivePerGroupClusters(rolloutStrategy, existingClusterStatus) default: return nil, RolloutResult{}, fmt.Errorf("incorrect rollout strategy type %v", rolloutStrategy.Type) } } -func (r *RolloutHandler) getRolloutAllClusters(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +func (r *RolloutHandler[T]) getRolloutAllClusters(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { // Prepare the rollout strategy strategy := RolloutStrategy{Type: All} strategy.All = rolloutStrategy.All.DeepCopy() @@ -110,15 +116,18 @@ func (r *RolloutHandler) getRolloutAllClusters(rolloutStrategy RolloutStrategy, return &strategy, RolloutResult{}, err } - // Get all clusters and perform progressive rollout - totalClusterGroups := r.pdTracker.ExistingClusterGroupsBesides() - totalClusters := totalClusterGroups.GetClusters().UnsortedList() - rolloutResult := progressivePerCluster(totalClusterGroups, len(totalClusters), failureTimeout, statusFunc) + allClusterGroups := r.pdTracker.ExistingClusterGroupsBesides() + allClusters := allClusterGroups.GetClusters().UnsortedList() + + // Check for removed Clusters + currentClusterStatus, removedClusterStatus := r.getRemovedClusters(allClusterGroups, existingClusterStatus) + rolloutResult := progressivePerCluster(allClusterGroups, len(allClusters), failureTimeout, currentClusterStatus) + rolloutResult.ClustersRemoved = removedClusterStatus return &strategy, rolloutResult, nil } -func (r *RolloutHandler) getProgressiveClusters(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +func (r *RolloutHandler[T]) getProgressiveClusters(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { // Prepare the rollout strategy strategy := RolloutStrategy{Type: Progressive} strategy.Progressive = rolloutStrategy.Progressive.DeepCopy() @@ -126,37 +135,45 @@ func (r *RolloutHandler) getProgressiveClusters(rolloutStrategy RolloutStrategy, strategy.Progressive = &RolloutProgressive{} } - // Upgrade mandatory decision groups first - groupKeys := decisionGroupsToGroupKeys(strategy.Progressive.MandatoryDecisionGroups.MandatoryDecisionGroups) - clusterGroups := r.pdTracker.ExistingClusterGroups(groupKeys...) - - // Perform progressive rollout for mandatory decision groups - rolloutResult := progressivePerGroup(clusterGroups, maxTimeDuration, statusFunc) - if len(rolloutResult.ClustersToRollout) > 0 { - return &strategy, rolloutResult, nil - } - // Parse timeout for non-mandatory decision groups failureTimeout, err := parseTimeout(strategy.Progressive.Timeout.Timeout) if err != nil { return &strategy, RolloutResult{}, err } - // Calculate the length for progressive rollout - totalClusters := r.pdTracker.ExistingClusterGroupsBesides().GetClusters() - length, err := calculateLength(strategy.Progressive.MaxConcurrency, len(totalClusters)) + // Check for removed clusters + clusterGroups := r.pdTracker.ExistingClusterGroupsBesides() + currentClusterStatus, removedClusterStatus := r.getRemovedClusters(clusterGroups, existingClusterStatus) + + // Upgrade mandatory decision groups first + groupKeys := decisionGroupsToGroupKeys(strategy.Progressive.MandatoryDecisionGroups.MandatoryDecisionGroups) + clusterGroups = r.pdTracker.ExistingClusterGroups(groupKeys...) + + // Perform progressive rollOut for mandatory decision groups first. + if len(clusterGroups) > 0 { + rolloutResult := progressivePerGroup(clusterGroups, failureTimeout, currentClusterStatus) + if len(rolloutResult.ClustersToRollout) > 0 || len(rolloutResult.ClustersTimeOut) > 0 { + rolloutResult.ClustersRemoved = removedClusterStatus + return &strategy, rolloutResult, nil + } + } + + // Calculate the size of progressive rollOut + // If the MaxConcurrency not defined, total clusters length is considered as maxConcurrency. + clusterGroups = r.pdTracker.ExistingClusterGroupsBesides(groupKeys...) + length, err := calculateRolloutSize(strategy.Progressive.MaxConcurrency, len(clusterGroups.GetClusters())) if err != nil { return &strategy, RolloutResult{}, err } - // Upgrade the remaining clusters - restClusterGroups := r.pdTracker.ExistingClusterGroupsBesides(clusterGroups.GetOrderedGroupKeys()...) - rolloutResult = progressivePerCluster(restClusterGroups, length, failureTimeout, statusFunc) + // Rollout the remaining clusters + rolloutResult := progressivePerCluster(clusterGroups, length, failureTimeout, currentClusterStatus) + rolloutResult.ClustersRemoved = removedClusterStatus return &strategy, rolloutResult, nil } -func (r *RolloutHandler) getProgressivePerGroupClusters(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +func (r *RolloutHandler[T]) getProgressivePerGroupClusters(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { // Prepare the rollout strategy strategy := RolloutStrategy{Type: ProgressivePerGroup} strategy.ProgressivePerGroup = rolloutStrategy.ProgressivePerGroup.DeepCopy() @@ -164,65 +181,99 @@ func (r *RolloutHandler) getProgressivePerGroupClusters(rolloutStrategy RolloutS strategy.ProgressivePerGroup = &RolloutProgressivePerGroup{} } + // Parse timeout for non-mandatory decision groups + failureTimeout, err := parseTimeout(strategy.ProgressivePerGroup.Timeout.Timeout) + if err != nil { + return &strategy, RolloutResult{}, err + } + + // Check for removed Clusters + clusterGroups := r.pdTracker.ExistingClusterGroupsBesides() + currentClusterStatus, removedClusterStatus := r.getRemovedClusters(clusterGroups, existingClusterStatus) + // Upgrade mandatory decision groups first mandatoryDecisionGroups := strategy.ProgressivePerGroup.MandatoryDecisionGroups.MandatoryDecisionGroups groupKeys := decisionGroupsToGroupKeys(mandatoryDecisionGroups) - clusterGroups := r.pdTracker.ExistingClusterGroups(groupKeys...) + clusterGroups = r.pdTracker.ExistingClusterGroups(groupKeys...) - // Perform progressive rollout per group for mandatory decision groups - rolloutResult := progressivePerGroup(clusterGroups, maxTimeDuration, statusFunc) - if len(rolloutResult.ClustersToRollout) > 0 { - return &strategy, rolloutResult, nil - } + // Perform progressive rollout per group for mandatory decision groups first + if len(clusterGroups) > 0 { + rolloutResult := progressivePerGroup(clusterGroups, failureTimeout, currentClusterStatus) - // Parse timeout for non-mandatory decision groups - failureTimeout, err := parseTimeout(strategy.ProgressivePerGroup.Timeout.Timeout) - if err != nil { - return &strategy, RolloutResult{}, err + if len(rolloutResult.ClustersToRollout) > 0 || len(rolloutResult.ClustersTimeOut) > 0 { + rolloutResult.ClustersRemoved = removedClusterStatus + return &strategy, rolloutResult, nil + } } - // Upgrade the rest of the decision groups - restClusterGroups := r.pdTracker.ExistingClusterGroupsBesides(clusterGroups.GetOrderedGroupKeys()...) + // RollOut the rest of the decision groups + restClusterGroups := r.pdTracker.ExistingClusterGroupsBesides(groupKeys...) // Perform progressive rollout per group for the remaining decision groups - rolloutResult = progressivePerGroup(restClusterGroups, failureTimeout, statusFunc) + rolloutResult := progressivePerGroup(restClusterGroups, failureTimeout, currentClusterStatus) + rolloutResult.ClustersRemoved = removedClusterStatus + return &strategy, rolloutResult, nil } -func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, length int, timeout time.Duration, statusFunc ClusterRolloutStatusFunc) RolloutResult { - rolloutClusters := map[string]ClusterRolloutStatus{} - timeoutClusters := map[string]ClusterRolloutStatus{} +func (r *RolloutHandler[T]) getRemovedClusters(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, existingClusterStatus []ClusterRolloutStatus) ([]ClusterRolloutStatus, []ClusterRolloutStatus) { + var currentClusterStatus, removedClusterStatus []ClusterRolloutStatus - if length == 0 { - return RolloutResult{ - ClustersToRollout: rolloutClusters, - ClustersTimeOut: timeoutClusters, + clusters := clusterGroupsMap.GetClusters().UnsortedList() + for _, clusterStatus := range existingClusterStatus { + exist := false + for _, cluster := range clusters { + if clusterStatus.ClusterName == cluster { + exist = true + currentClusterStatus = append(currentClusterStatus, clusterStatus) + break + } + } + + if !exist { + removedClusterStatus = append(removedClusterStatus, clusterStatus) + } + } + return currentClusterStatus, removedClusterStatus +} + +func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, length int, timeout time.Duration, existingClusterStatus []ClusterRolloutStatus) RolloutResult { + var rolloutClusters, timeoutClusters []ClusterRolloutStatus + existingClusters := make(map[string]bool) + + for _, status := range existingClusterStatus { + if status.ClusterName == "" { + continue + } + + existingClusters[status.ClusterName] = true + rolloutClusters, timeoutClusters = determineRolloutStatus(status, timeout, rolloutClusters, timeoutClusters) + + if len(rolloutClusters) >= length { + return RolloutResult{ + ClustersToRollout: rolloutClusters, + ClustersTimeOut: timeoutClusters, + } } } clusters := clusterGroupsMap.GetClusters().UnsortedList() clusterToGroupKey := clusterGroupsMap.ClusterToGroupKey() - // Sort the clusters in alphabetical order to ensure consistency. sort.Strings(clusters) for _, cluster := range clusters { - status := statusFunc(cluster) - if groupKey, exists := clusterToGroupKey[cluster]; exists { - status.GroupKey = groupKey + if existingClusters[cluster] { + continue } - newStatus, needToRollout := determineRolloutStatusAndContinue(status, timeout) - status.Status = newStatus.Status - status.TimeOutTime = newStatus.TimeOutTime - - if needToRollout { - rolloutClusters[cluster] = status - } - if status.Status == TimeOut { - timeoutClusters[cluster] = status + status := ClusterRolloutStatus{ + ClusterName: cluster, + Status: ToApply, + GroupKey: clusterToGroupKey[cluster], } + rolloutClusters = append(rolloutClusters, status) - if len(rolloutClusters)%length == 0 && len(rolloutClusters) > 0 { + if len(rolloutClusters) >= length { return RolloutResult{ ClustersToRollout: rolloutClusters, ClustersTimeOut: timeoutClusters, @@ -236,32 +287,44 @@ func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, len } } -func progressivePerGroup(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, timeout time.Duration, statusFunc ClusterRolloutStatusFunc) RolloutResult { - rolloutClusters := map[string]ClusterRolloutStatus{} - timeoutClusters := map[string]ClusterRolloutStatus{} +func progressivePerGroup(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, timeout time.Duration, existingClusterStatus []ClusterRolloutStatus) RolloutResult { + var rolloutClusters, timeoutClusters []ClusterRolloutStatus + existingClusters := make(map[string]bool) - clusterGroupKeys := clusterGroupsMap.GetOrderedGroupKeys() + for _, status := range existingClusterStatus { + if status.ClusterName == "" { + continue + } + + if status.Status == ToApply { + // Set as false to consider the cluster in the decisionGroups iteration. + existingClusters[status.ClusterName] = false + } else { + existingClusters[status.ClusterName] = true + rolloutClusters, timeoutClusters = determineRolloutStatus(status, timeout, rolloutClusters, timeoutClusters) + } + } + clusterGroupKeys := clusterGroupsMap.GetOrderedGroupKeys() for _, key := range clusterGroupKeys { if subclusters, ok := clusterGroupsMap[key]; ok { // Iterate through clusters in the group - for _, cluster := range subclusters.UnsortedList() { - status := statusFunc(cluster) - status.GroupKey = key - - newStatus, needToRollout := determineRolloutStatusAndContinue(status, timeout) - status.Status = newStatus.Status - status.TimeOutTime = newStatus.TimeOutTime - - if needToRollout { - rolloutClusters[cluster] = status + clusters := subclusters.UnsortedList() + sort.Strings(clusters) + for _, cluster := range clusters { + if existingClusters[cluster] { + continue } - if status.Status == TimeOut { - timeoutClusters[cluster] = status + + status := ClusterRolloutStatus{ + ClusterName: cluster, + Status: ToApply, + GroupKey: key, } + rolloutClusters = append(rolloutClusters, status) } - // Return if there are clusters to rollout + // As it is perGroup Return if there are clusters to rollOut if len(rolloutClusters) > 0 { return RolloutResult{ ClustersToRollout: rolloutClusters, @@ -277,36 +340,33 @@ func progressivePerGroup(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, timeo } } -// determineRolloutStatusAndContinue checks whether a cluster should continue its rollout based on -// its current status and timeout. The function returns an updated cluster status and a boolean -// indicating whether the rollout should continue. +// determineRolloutStatus checks whether a cluster should continue its rollout based on its current status and timeout. +// The function update the cluster status and append it to the expected slice. // -// The timeout parameter is utilized for handling progressing and failed statuses: -// 1. If timeout is set to None (maxTimeDuration), the function will wait until cluster reaching a success status. -// It returns true to include the cluster in the result and halts the rollout of other clusters or groups. -// 2. If timeout is set to 0, the function proceeds with upgrading other clusters without waiting. -// It returns false to skip waiting for the cluster to reach a success status and continues to rollout others. -func determineRolloutStatusAndContinue(status ClusterRolloutStatus, timeout time.Duration) (*ClusterRolloutStatus, bool) { - newStatus := status.DeepCopy() +// The timeout parameter is utilized for handling progressing and failed statuses and any other unknown status: +// 1. If timeout is set to None (maxTimeDuration), the function will append the clusterStatus to the rollOut Clusters. +// 2. If timeout is set to 0, the function append the clusterStatus to the timeOut clusters. +func determineRolloutStatus(status ClusterRolloutStatus, timeout time.Duration, rolloutClusters []ClusterRolloutStatus, timeoutClusters []ClusterRolloutStatus) ([]ClusterRolloutStatus, []ClusterRolloutStatus) { + switch status.Status { case ToApply: - return newStatus, true + rolloutClusters = append(rolloutClusters, status) case TimeOut, Succeeded, Skip: - return newStatus, false - case Progressing, Failed: + return rolloutClusters, timeoutClusters + default: // For progressing, failed status and any other unknown status. timeOutTime := getTimeOutTime(status.LastTransitionTime, timeout) - newStatus.TimeOutTime = timeOutTime + status.TimeOutTime = timeOutTime // check if current time is before the timeout time if RolloutClock.Now().Before(timeOutTime.Time) { - return newStatus, true + rolloutClusters = append(rolloutClusters, status) } else { - newStatus.Status = TimeOut - return newStatus, false + status.Status = TimeOut + timeoutClusters = append(timeoutClusters, status) } - default: - return newStatus, true } + + return rolloutClusters, timeoutClusters } // get the timeout time @@ -320,7 +380,7 @@ func getTimeOutTime(startTime *metav1.Time, timeout time.Duration) *metav1.Time return &metav1.Time{Time: timeoutTime} } -func calculateLength(maxConcurrency intstr.IntOrString, total int) (int, error) { +func calculateRolloutSize(maxConcurrency intstr.IntOrString, total int) (int, error) { length := total switch maxConcurrency.Type { diff --git a/cluster/v1alpha1/helpers_test.go b/cluster/v1alpha1/helpers_test.go index badcb6b28..396e3bd9e 100644 --- a/cluster/v1alpha1/helpers_test.go +++ b/cluster/v1alpha1/helpers_test.go @@ -16,7 +16,6 @@ import ( ) var fakeTime = metav1.NewTime(time.Date(2022, time.January, 01, 0, 0, 0, 0, time.UTC)) -var fakeTimeMax = metav1.NewTime(fakeTime.Add(maxTimeDuration)) var fakeTimeMax_60s = metav1.NewTime(fakeTime.Add(maxTimeDuration - time.Minute)) var fakeTimeMax_120s = metav1.NewTime(fakeTime.Add(maxTimeDuration - 2*time.Minute)) var fakeTime30s = metav1.NewTime(fakeTime.Add(30 * time.Second)) @@ -28,100 +27,125 @@ type FakePlacementDecisionGetter struct { FakeDecisions []*clusterv1beta1.PlacementDecision } +// Dummy workload type that will be used to create a RolloutHandler. +type dummyWorkload struct { + ClusterGroup clusterv1beta1.GroupKey + ClusterName string + State string + LastTransitionTime *metav1.Time +} + +// Dummy Workload status +const ( + valid = "valid" + applying = "applying" + done = "done" + missing = "missing" +) + +// Dummy ClusterRolloutStatusFunc implementation that will be used to create a RolloutHandler. +func dummyWorkloadClusterRolloutStatusFunc(clusterName string, workload dummyWorkload) (ClusterRolloutStatus, error) { + // workload obj should be used to determine the clusterRolloutStatus. + switch workload.State { + case valid: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: ToApply, LastTransitionTime: workload.LastTransitionTime}, nil + case applying: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: Progressing, LastTransitionTime: workload.LastTransitionTime}, nil + case done: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: Succeeded, LastTransitionTime: workload.LastTransitionTime}, nil + case missing: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: Failed, LastTransitionTime: workload.LastTransitionTime}, nil + default: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: ToApply, LastTransitionTime: workload.LastTransitionTime}, nil + } +} + +type testCase struct { + name string + rolloutStrategy RolloutStrategy + existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] + clusterRolloutStatusFunc ClusterRolloutStatusFunc[dummyWorkload] // Using type dummy workload obj + expectRolloutStrategy *RolloutStrategy + existingWorkloads []dummyWorkload + expectRolloutClusters []ClusterRolloutStatus + expectTimeOutClusters []ClusterRolloutStatus + expectRemovedClusters []ClusterRolloutStatus +} + func (f *FakePlacementDecisionGetter) List(selector labels.Selector, namespace string) (ret []*clusterv1beta1.PlacementDecision, err error) { return f.FakeDecisions, nil } func TestGetRolloutCluster_All(t *testing.T) { - tests := []struct { - name string - rolloutStrategy RolloutStrategy - existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] - clusterRolloutStatusFunc ClusterRolloutStatusFunc - expectRolloutStrategy *RolloutStrategy - expectRolloutClusters map[string]ClusterRolloutStatus - expectTimeOutClusters map[string]ClusterRolloutStatus - }{ + tests := []testCase{ { - name: "test rollout all with timeout 90s", + name: "test rollout all with timeout 90s witout workload created", rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Progressing, LastTransitionTime: &fakeTime_60s}, - "cluster3": {Status: Succeeded, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] - }, - expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster6": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, - }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + existingWorkloads: []dummyWorkload{}, + expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, }, { - name: "test rollout all (default timeout None)", - rolloutStrategy: RolloutStrategy{Type: All}, + name: "test rollout all with timeout 90s", + rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] - }, - expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{""}}}, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, TimeOutTime: &fakeTimeMax}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, TimeOutTime: &fakeTimeMax}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, - }, - { - name: "test rollout all with timeout 0s", - rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"0s"}}}, - existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster4", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster5", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, }, - expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"0s"}}}, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, TimeOutTime: &fakeTime}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, TimeOutTime: &fakeTime}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, }, } @@ -132,8 +156,14 @@ func TestGetRolloutCluster_All(t *testing.T) { fakeGetter := FakePlacementDecisionGetter{} tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) - rolloutHandler, _ := NewRolloutHandler(tracker) - actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, test.clusterRolloutStatusFunc) + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) if !reflect.DeepEqual(actualRolloutStrategy.All, test.expectRolloutStrategy.All) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) @@ -147,136 +177,316 @@ func TestGetRolloutCluster_All(t *testing.T) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect timeout clusters: %v, actual : %v", test.name, test.expectTimeOutClusters, actualRolloutResult.ClustersTimeOut) return } + if !reflect.DeepEqual(actualRolloutResult.ClustersRemoved, test.expectRemovedClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect removed clusters: %v, actual : %v", test.name, test.expectRemovedClusters, actualRolloutResult.ClustersRemoved) + return + } } } func TestGetRolloutCluster_Progressive(t *testing.T) { - tests := []struct { - name string - rolloutStrategy RolloutStrategy - existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] - clusterRolloutStatusFunc ClusterRolloutStatusFunc - expectRolloutStrategy *RolloutStrategy - expectRolloutClusters map[string]ClusterRolloutStatus - expectTimeOutClusters map[string]ClusterRolloutStatus - }{ + tests := []testCase{ { - name: "test progressive rollout with timeout 90s", + name: "test progressive rollout with timeout 90s witout workload created", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{"90s"}, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Progressing, LastTransitionTime: &fakeTime_60s}, - "cluster3": {Status: Succeeded, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), + }, + }, + existingWorkloads: []dummyWorkload{}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + }, + }, + { + name: "test progressive rollout with timeout 90s and workload clusterRollOutStatus are in ToApply status", + rolloutStrategy: RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + MaxConcurrency: intstr.FromInt(4), + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + MaxConcurrency: intstr.FromInt(4), + Timeout: Timeout{"90s"}, + }, }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster4", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster5", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster6", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster7", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster8", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster9", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster10", + State: valid, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + }, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster10", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, + }, + }, + { + name: "test progressive rollout with timeout 90s and MaxConcurrency not set", + rolloutStrategy: RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ Timeout: Timeout{"90s"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster6": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster4", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster5", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster6", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster7", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster8", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster9", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster10", + State: valid, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster7", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, + {ClusterName: "cluster8", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, + {ClusterName: "cluster9", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster10", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, }, }, { - name: "test progressive rollout with timeout None and MaxConcurrency 50%", + name: "test progressive rollout with timeout 90s", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{""}, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{""}, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, TimeOutTime: &fakeTimeMax}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, TimeOutTime: &fakeTimeMax}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive rollout with timeout 0s and MaxConcurrency 3", + name: "test progressive rollout with timeout 0s", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ Timeout: Timeout{"0s"}, - MaxConcurrency: intstr.FromInt(3), // Maximum 3 clusters concurrently + MaxConcurrency: intstr.FromInt(2), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ Timeout: Timeout{"0s"}, - MaxConcurrency: intstr.FromInt(3), // Maximum 3 clusters concurrently + MaxConcurrency: intstr.FromInt(2), + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_30s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: applying, + LastTransitionTime: &fakeTime_30s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_30s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, TimeOutTime: &fakeTime}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, TimeOutTime: &fakeTime}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime_30s}, }, }, { - name: "test progressive rollout with mandatory decision groups", + name: "test progressive rollout with mandatroyDecisionGroup and timeout 90s ", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -285,23 +495,15 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromString("50%"), + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(3), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: ToApply}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -310,18 +512,86 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromString("50%"), - Timeout: Timeout{""}, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(3), + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive rollout with mandatory decision groups Succeed", + name: "test progressive rollout with timeout None and MaxConcurrency 50%", + rolloutStrategy: RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + Timeout: Timeout{"None"}, + MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + Timeout: Timeout{"None"}, + MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTimeMax_120s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTimeMax_60s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + }, + }, + { + name: "test progressive rollout with mandatory decision groups failed", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -330,24 +600,16 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromInt(2), + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Succeeded}, - "cluster2": {Status: Succeeded}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: Succeeded}, - "cluster5": {Status: ToApply}, - "cluster6": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -356,18 +618,33 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromInt(2), - Timeout: Timeout{""}, + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: missing, + LastTransitionTime: &fakeTime_120s, }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster3": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive rollout with mandatory decision groups failed", + name: "test progressive rollout with mandatory decision groups Succeed", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -376,24 +653,15 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters - Timeout: Timeout{"0s"}, + MaxConcurrency: intstr.FromInt(3), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed}, - "cluster2": {Status: Failed}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -402,15 +670,29 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters - Timeout: Timeout{"0s"}, + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{""}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_120s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 2}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, } @@ -422,8 +704,14 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { fakeGetter := FakePlacementDecisionGetter{} tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) - rolloutHandler, _ := NewRolloutHandler(tracker) - actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, test.clusterRolloutStatusFunc) + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) if !reflect.DeepEqual(actualRolloutStrategy.Progressive, test.expectRolloutStrategy.Progressive) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) @@ -437,21 +725,17 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect timeout clusters: %v, actual : %v", test.name, test.expectTimeOutClusters, actualRolloutResult.ClustersTimeOut) return } + if !reflect.DeepEqual(actualRolloutResult.ClustersRemoved, test.expectRemovedClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect removed clusters: %v, actual : %v", test.name, test.expectRemovedClusters, actualRolloutResult.ClustersRemoved) + return + } } } func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { - tests := []struct { - name string - rolloutStrategy RolloutStrategy - existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] - clusterRolloutStatusFunc ClusterRolloutStatusFunc - expectRolloutStrategy *RolloutStrategy - expectRolloutClusters map[string]ClusterRolloutStatus - expectTimeOutClusters map[string]ClusterRolloutStatus - }{ + tests := []testCase{ { - name: "test progressive per group rollout with timeout 90s", + name: "test progressivePerGroup rollout with timeout 90s witout workload created", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -459,191 +743,289 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster3": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ Timeout: Timeout{"90s"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + existingWorkloads: []dummyWorkload{}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, }, { - name: "test progressive per group rollout with timeout None", + name: "test progressivePerGroup rollout with timeout 90s and all workload clusterRollOutStatus are in ToApply status", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster3": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - Timeout: Timeout{""}, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster4", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster5", + State: valid, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster6", + State: valid, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTimeMax_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTimeMax_120s}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive per group rollout with timeout 0s", + name: "test progressivePerGroup rollout with timeout 90s", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster3": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster3": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_120s}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, }, { - name: "test progressive per group rollout with mandatory decision groups", + name: "test progressivePerGroup rollout with timeout 90s and first group timeOut", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: ToApply}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive per group rollout with mandatory decision groups Succeed", + name: "test progressivePerGroup rollout with timeout 90s and first group timeOut, second group successed", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4"), - {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Succeeded}, - "cluster2": {Status: Succeeded}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: Succeeded}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, + Timeout: Timeout{"90s"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster3": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster4", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster5", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster6", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster7", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, + {ClusterName: "cluster8", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, + {ClusterName: "cluster9", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 2}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive per group rollout with mandatory decision groups failed", + name: "test progressivePerGroup rollout with timeout None and first group failing", + rolloutStrategy: RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"None"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"None"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTimeMax_120s}, + }, + }, + { + name: "test ProgressivePerGroup rollout with mandatroyDecisionGroup failing and timeout 90s ", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -652,23 +1034,14 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {GroupName: "group1"}, }, }, - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed}, - "cluster2": {Status: Failed}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -677,14 +1050,89 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {GroupName: "group1"}, }, }, - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: missing, + LastTransitionTime: &fakeTime_120s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + }, + }, + { + name: "test ProgressivePerGroup rollout with mandatroyDecisionGroup Succeeded and timeout 90s ", + rolloutStrategy: RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, } @@ -696,8 +1144,14 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { fakeGetter := FakePlacementDecisionGetter{} tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) - rolloutHandler, _ := NewRolloutHandler(tracker) - actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, test.clusterRolloutStatusFunc) + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) if !reflect.DeepEqual(actualRolloutStrategy.ProgressivePerGroup, test.expectRolloutStrategy.ProgressivePerGroup) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) @@ -711,90 +1165,658 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect timeout clusters: %v, actual : %v", test.name, test.expectTimeOutClusters, actualRolloutResult.ClustersTimeOut) return } + if !reflect.DeepEqual(actualRolloutResult.ClustersRemoved, test.expectRemovedClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect removed clusters: %v, actual : %v", test.name, test.expectRemovedClusters, actualRolloutResult.ClustersRemoved) + return + } + } +} + +func TestGetRolloutCluster_ClusterAdded(t *testing.T) { + tests := []testCase{ + { + name: "test rollout all with timeout 90s and cluster added", + rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster7"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster4", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster5", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster7", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + }, + }, + { + name: "test progressive rollout with mandatory decision groups Succeed and clusters added after rollout", + rolloutStrategy: RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster7"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster8"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster5", "cluster6", "cluster9"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster4", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 2}, + ClusterName: "cluster5", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + //{ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 2}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster7", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + }, + }, + { + name: "test progressivePerGroup rollout with timeout 90s and cluster added after rollout start.", + rolloutStrategy: RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster4", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster5", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + }, + }, + } + + // Set the fake time for testing + RolloutClock = testingclock.NewFakeClock(fakeTime.Time) + + for _, test := range tests { + // Init fake placement decision tracker + fakeGetter := FakePlacementDecisionGetter{} + tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) + + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) + + if !reflect.DeepEqual(actualRolloutStrategy.Type, test.expectRolloutStrategy.Type) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersToRollout, test.expectRolloutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect rollout clusters: %v, actual : %v", test.name, test.expectRolloutClusters, actualRolloutResult.ClustersToRollout) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersTimeOut, test.expectTimeOutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect timeout clusters: %v, actual : %v", test.name, test.expectTimeOutClusters, actualRolloutResult.ClustersTimeOut) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersRemoved, test.expectRemovedClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect removed clusters: %v, actual : %v", test.name, test.expectRemovedClusters, actualRolloutResult.ClustersRemoved) + return + } + } +} + +func TestGetRolloutCluster_ClusterRemoved(t *testing.T) { + tests := []testCase{ + { + name: "test rollout all with timeout 90s and clusters removed", + rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster5"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster4", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster5", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + }, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s}, + }, + }, + { + name: "test progressive rollout with timeout 90s and cluster removed", + rolloutStrategy: RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + }, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Succeeded, LastTransitionTime: &fakeTime_60s}, + }, + }, + { + name: "test progressive rollout with mandatroyDecisionGroup, timeout 90s and cluster removed from mandatroyDecisionGroup", + rolloutStrategy: RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: Progressive, + Progressive: &RolloutProgressive{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + }, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s}, + }, + }, + { + name: "test progressivePerGroup rollout with timeout 90s and cluster removed after rollout start.", + rolloutStrategy: RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + }, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s}, + }, + }, + { + name: "test progressivePerGroup rollout with timeout 90s and cluster removed after rollout start while the group timeout.", + rolloutStrategy: RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_120s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + }, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s}, + }, + }, + { + name: "test ProgressivePerGroup rollout with mandatroyDecisionGroup, timeout 90s and cluster removed from mandatroyDecisionGroup", + rolloutStrategy: RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + }, + expectRemovedClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s}, + }, + }, + } + + // Set the fake time for testing + RolloutClock = testingclock.NewFakeClock(fakeTime.Time) + + for _, test := range tests { + // Init fake placement decision tracker + fakeGetter := FakePlacementDecisionGetter{} + tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) + + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) + + if !reflect.DeepEqual(actualRolloutStrategy.Type, test.expectRolloutStrategy.Type) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersToRollout, test.expectRolloutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect rollout clusters: %v, actual : %v", test.name, test.expectRolloutClusters, actualRolloutResult.ClustersToRollout) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersTimeOut, test.expectTimeOutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect timeout clusters: %v, actual : %v", test.name, test.expectTimeOutClusters, actualRolloutResult.ClustersTimeOut) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersRemoved, test.expectRemovedClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect removed clusters: %v, actual : %v", test.name, test.expectRemovedClusters, actualRolloutResult.ClustersRemoved) + return + } } + } -func TestNeedToUpdate(t *testing.T) { +func TestDetermineRolloutStatus(t *testing.T) { testCases := []struct { - name string - status RolloutStatus - lastTransition *metav1.Time - timeout time.Duration - expectedResult bool + name string + timeout time.Duration + clusterStatus ClusterRolloutStatus + expectRolloutClusters []ClusterRolloutStatus + expectTimeOutClusters []ClusterRolloutStatus }{ { - name: "ToApply status", - status: ToApply, - lastTransition: nil, - timeout: time.Minute, - expectedResult: true, + name: "ToApply status", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: ToApply}, + timeout: time.Minute, + expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: ToApply}}, + }, + { + name: "Skip status", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Skip}, + timeout: time.Minute, }, { - name: "Progressing status", - status: Progressing, - lastTransition: nil, - timeout: time.Minute, - expectedResult: true, + name: "Succeeded status", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Succeeded}, + timeout: time.Minute, }, { - name: "Succeeded status", - status: Succeeded, - lastTransition: nil, - timeout: time.Minute, - expectedResult: false, + name: "TimeOut status", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: TimeOut}, + timeout: time.Minute, }, { - name: "Failed status, timeout is None", - status: Failed, - lastTransition: &fakeTime, - timeout: maxTimeDuration, - expectedResult: true, + name: "Progressing status within the timeout duration", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Progressing, LastTransitionTime: &fakeTime_30s}, + timeout: time.Minute, + expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: Progressing, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime30s}}, }, { - name: "Failed status, timeout is 0", - status: Failed, - lastTransition: &fakeTime, - timeout: 0, - expectedResult: false, + name: "Failed status out the timeout duration", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: Failed, LastTransitionTime: &fakeTime_60s}, + timeout: time.Minute, + expectTimeOutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: TimeOut, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime}}, }, { - name: "Failed status, within the timeout duration", - status: Failed, - lastTransition: &fakeTime_60s, - timeout: 2 * time.Minute, - expectedResult: true, + name: "unknown status out the timeout duration", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: 8, LastTransitionTime: &fakeTime_60s}, + timeout: time.Minute, + expectTimeOutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: TimeOut, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime}}, }, { - name: "Failed status, outside the timeout duration", - status: Failed, - lastTransition: &fakeTime_120s, - timeout: time.Minute, - expectedResult: false, + name: "unknown status within the timeout duration", + clusterStatus: ClusterRolloutStatus{ClusterName: "cluster1", Status: 9, LastTransitionTime: &fakeTime_30s}, + timeout: time.Minute, + expectRolloutClusters: []ClusterRolloutStatus{{ClusterName: "cluster1", Status: 9, LastTransitionTime: &fakeTime_30s, TimeOutTime: &fakeTime30s}}, }, } RolloutClock = testingclock.NewFakeClock(fakeTime.Time) - // Run the tests for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Create a ClusterRolloutStatus instance - status := ClusterRolloutStatus{ - Status: tc.status, - LastTransitionTime: tc.lastTransition, - } - - // Call the determineRolloutStatusAndContinue function - _, result := determineRolloutStatusAndContinue(status, tc.timeout) - - // Compare the result with the expected result - if result != tc.expectedResult { - t.Errorf("Expected result: %v, got: %v", tc.expectedResult, result) - } - }) + var rolloutClusters, timeoutClusters []ClusterRolloutStatus + rolloutClusters, timeoutClusters = determineRolloutStatus(tc.clusterStatus, tc.timeout, rolloutClusters, timeoutClusters) + if !reflect.DeepEqual(rolloutClusters, tc.expectRolloutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect rollout clusters: %v, actual : %v", tc.name, tc.expectRolloutClusters, rolloutClusters) + return + } + if !reflect.DeepEqual(timeoutClusters, tc.expectTimeOutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect timeout clusters: %v, actual : %v", tc.name, tc.expectTimeOutClusters, timeoutClusters) + return + } } } -func TestCalculateLength(t *testing.T) { +func TestCalculateRolloutSize(t *testing.T) { total := 100 tests := []struct { @@ -813,7 +1835,7 @@ func TestCalculateLength(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - length, err := calculateLength(test.maxConcurrency, total) + length, err := calculateRolloutSize(test.maxConcurrency, total) // Compare the result with the expected result if length != test.expected { diff --git a/cluster/v1alpha1/zz_generated.deepcopy.go b/cluster/v1alpha1/zz_generated.deepcopy.go index a72d4c816..00938a885 100644 --- a/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/cluster/v1alpha1/zz_generated.deepcopy.go @@ -315,14 +315,14 @@ func (in *RolloutResult) DeepCopyInto(out *RolloutResult) { *out = *in if in.ClustersToRollout != nil { in, out := &in.ClustersToRollout, &out.ClustersToRollout - *out = make(map[string]ClusterRolloutStatus, len(*in)) + *out = make([]ClusterRolloutStatus, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() } } if in.ClustersTimeOut != nil { in, out := &in.ClustersTimeOut, &out.ClustersTimeOut - *out = make(map[string]ClusterRolloutStatus, len(*in)) + *out = make([]ClusterRolloutStatus, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() }