diff --git a/cluster/v1alpha1/helpers.go b/cluster/v1alpha1/helpers.go index d70865825..e4d4aad30 100644 --- a/cluster/v1alpha1/helpers.go +++ b/cluster/v1alpha1/helpers.go @@ -37,11 +37,10 @@ const ( Skip ) -// ClusterRolloutStatusFunc defines a function to return the rollout status for a managed cluster. -type ClusterRolloutStatusFunc func(clusterName string) ClusterRolloutStatus - // ClusterRolloutStatus holds the rollout status information for a cluster. type ClusterRolloutStatus struct { + // cluster name + ClusterName string // GroupKey represents the cluster group key (optional field). GroupKey clusterv1beta1.GroupKey // Status is the required field indicating the rollout status. @@ -56,23 +55,29 @@ type ClusterRolloutStatus struct { // RolloutResult contains the clusters to be rolled out and the clusters that have timed out. type RolloutResult struct { // ClustersToRollout is a map where the key is the cluster name and the value is the ClusterRolloutStatus. - ClustersToRollout map[string]ClusterRolloutStatus + ClustersToRollout []ClusterRolloutStatus // ClustersTimeOut is a map where the key is the cluster name and the value is the ClusterRolloutStatus. - ClustersTimeOut map[string]ClusterRolloutStatus + ClustersTimeOut []ClusterRolloutStatus + // ClustersRemoved is a map where the key is the cluster name and the value is the ClusterRolloutStatus. + ClustersRemoved []ClusterRolloutStatus } +// ClusterRolloutStatusFunc defines a function to return the rollout status for a managed cluster. +type ClusterRolloutStatusFunc[T any] func(clusterName string, workload T) (ClusterRolloutStatus, error) + // +k8s:deepcopy-gen=false -type RolloutHandler struct { +type RolloutHandler[T any] struct { // placement decision tracker - pdTracker *clusterv1beta1.PlacementDecisionClustersTracker + pdTracker *clusterv1beta1.PlacementDecisionClustersTracker + statusFunc ClusterRolloutStatusFunc[T] } -func NewRolloutHandler(pdTracker *clusterv1beta1.PlacementDecisionClustersTracker) (*RolloutHandler, error) { +func NewRolloutHandler[T any](pdTracker *clusterv1beta1.PlacementDecisionClustersTracker, statusFunc ClusterRolloutStatusFunc[T]) (*RolloutHandler[T], error) { if pdTracker == nil { return nil, fmt.Errorf("invalid placement decision tracker %v", pdTracker) } - return &RolloutHandler{pdTracker: pdTracker}, nil + return &RolloutHandler[T]{pdTracker: pdTracker, statusFunc: statusFunc}, nil } // The input is a duck type RolloutStrategy and a ClusterRolloutStatusFunc to return the rollout status on each managed cluster. @@ -83,20 +88,21 @@ func NewRolloutHandler(pdTracker *clusterv1beta1.PlacementDecisionClustersTracke // // ClustersTimeOut: If the cluster status is Progressing or Failed, and the status lasts longer than timeout defined in strategy, // will list them RolloutResult.ClustersTimeOut with status TimeOut. -func (r *RolloutHandler) GetRolloutCluster(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +// func (r *RolloutHandler) GetRolloutCluster(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +func (r *RolloutHandler[T]) GetRolloutCluster(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { switch rolloutStrategy.Type { case All: - return r.getRolloutAllClusters(rolloutStrategy, statusFunc) + return r.getRolloutAllClusters(rolloutStrategy, existingClusterStatus) case Progressive: - return r.getProgressiveClusters(rolloutStrategy, statusFunc) + return r.getProgressiveClusters(rolloutStrategy, existingClusterStatus) case ProgressivePerGroup: - return r.getProgressivePerGroupClusters(rolloutStrategy, statusFunc) + return r.getProgressivePerGroupClusters(rolloutStrategy, existingClusterStatus) default: return nil, RolloutResult{}, fmt.Errorf("incorrect rollout strategy type %v", rolloutStrategy.Type) } } -func (r *RolloutHandler) getRolloutAllClusters(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +func (r *RolloutHandler[T]) getRolloutAllClusters(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { // Prepare the rollout strategy strategy := RolloutStrategy{Type: All} strategy.All = rolloutStrategy.All.DeepCopy() @@ -113,12 +119,12 @@ func (r *RolloutHandler) getRolloutAllClusters(rolloutStrategy RolloutStrategy, // Get all clusters and perform progressive rollout totalClusterGroups := r.pdTracker.ExistingClusterGroupsBesides() totalClusters := totalClusterGroups.GetClusters().UnsortedList() - rolloutResult := progressivePerCluster(totalClusterGroups, len(totalClusters), failureTimeout, statusFunc) + rolloutResult := progressivePerCluster(totalClusterGroups, len(totalClusters), failureTimeout, existingClusterStatus) return &strategy, rolloutResult, nil } -func (r *RolloutHandler) getProgressiveClusters(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +func (r *RolloutHandler[T]) getProgressiveClusters(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { // Prepare the rollout strategy strategy := RolloutStrategy{Type: Progressive} strategy.Progressive = rolloutStrategy.Progressive.DeepCopy() @@ -126,22 +132,25 @@ func (r *RolloutHandler) getProgressiveClusters(rolloutStrategy RolloutStrategy, strategy.Progressive = &RolloutProgressive{} } - // Upgrade mandatory decision groups first - groupKeys := decisionGroupsToGroupKeys(strategy.Progressive.MandatoryDecisionGroups.MandatoryDecisionGroups) - clusterGroups := r.pdTracker.ExistingClusterGroups(groupKeys...) - - // Perform progressive rollout for mandatory decision groups - rolloutResult := progressivePerGroup(clusterGroups, maxTimeDuration, statusFunc) - if len(rolloutResult.ClustersToRollout) > 0 { - return &strategy, rolloutResult, nil - } - // Parse timeout for non-mandatory decision groups failureTimeout, err := parseTimeout(strategy.Progressive.Timeout.Timeout) if err != nil { return &strategy, RolloutResult{}, err } + // Upgrade mandatory decision groups first + groupKeys := decisionGroupsToGroupKeys(strategy.Progressive.MandatoryDecisionGroups.MandatoryDecisionGroups) + clusterGroups := r.pdTracker.ExistingClusterGroups(groupKeys...) + + // Perform progressive rollOut for mandatory decision groups first. + if len(clusterGroups) > 0 { + rolloutResult := progressivePerGroup(clusterGroups, failureTimeout, existingClusterStatus) + //fmt.Println("progressivePerGroup ", rolloutResult.ClustersToRollout) + if len(rolloutResult.ClustersToRollout) > 0 || len(rolloutResult.ClustersTimeOut) > 0 { + return &strategy, rolloutResult, nil + } + } + // Calculate the length for progressive rollout totalClusters := r.pdTracker.ExistingClusterGroupsBesides().GetClusters() length, err := calculateLength(strategy.Progressive.MaxConcurrency, len(totalClusters)) @@ -151,12 +160,12 @@ func (r *RolloutHandler) getProgressiveClusters(rolloutStrategy RolloutStrategy, // Upgrade the remaining clusters restClusterGroups := r.pdTracker.ExistingClusterGroupsBesides(clusterGroups.GetOrderedGroupKeys()...) - rolloutResult = progressivePerCluster(restClusterGroups, length, failureTimeout, statusFunc) + rolloutResult := progressivePerCluster(restClusterGroups, length, failureTimeout, existingClusterStatus) return &strategy, rolloutResult, nil } -func (r *RolloutHandler) getProgressivePerGroupClusters(rolloutStrategy RolloutStrategy, statusFunc ClusterRolloutStatusFunc) (*RolloutStrategy, RolloutResult, error) { +func (r *RolloutHandler[T]) getProgressivePerGroupClusters(rolloutStrategy RolloutStrategy, existingClusterStatus []ClusterRolloutStatus) (*RolloutStrategy, RolloutResult, error) { // Prepare the rollout strategy strategy := RolloutStrategy{Type: ProgressivePerGroup} strategy.ProgressivePerGroup = rolloutStrategy.ProgressivePerGroup.DeepCopy() @@ -164,34 +173,39 @@ func (r *RolloutHandler) getProgressivePerGroupClusters(rolloutStrategy RolloutS strategy.ProgressivePerGroup = &RolloutProgressivePerGroup{} } + // Parse timeout for non-mandatory decision groups + failureTimeout, err := parseTimeout(strategy.ProgressivePerGroup.Timeout.Timeout) + if err != nil { + return &strategy, RolloutResult{}, err + } + // Upgrade mandatory decision groups first mandatoryDecisionGroups := strategy.ProgressivePerGroup.MandatoryDecisionGroups.MandatoryDecisionGroups groupKeys := decisionGroupsToGroupKeys(mandatoryDecisionGroups) clusterGroups := r.pdTracker.ExistingClusterGroups(groupKeys...) - // Perform progressive rollout per group for mandatory decision groups - rolloutResult := progressivePerGroup(clusterGroups, maxTimeDuration, statusFunc) - if len(rolloutResult.ClustersToRollout) > 0 { - return &strategy, rolloutResult, nil - } + // Perform progressive rollout per group for mandatory decision groups first + if len(clusterGroups) > 0 { + rolloutResult := progressivePerGroup(clusterGroups, failureTimeout, existingClusterStatus) - // Parse timeout for non-mandatory decision groups - failureTimeout, err := parseTimeout(strategy.ProgressivePerGroup.Timeout.Timeout) - if err != nil { - return &strategy, RolloutResult{}, err + if len(rolloutResult.ClustersToRollout) > 0 || len(rolloutResult.ClustersTimeOut) > 0 { + return &strategy, rolloutResult, nil + } } // Upgrade the rest of the decision groups restClusterGroups := r.pdTracker.ExistingClusterGroupsBesides(clusterGroups.GetOrderedGroupKeys()...) // Perform progressive rollout per group for the remaining decision groups - rolloutResult = progressivePerGroup(restClusterGroups, failureTimeout, statusFunc) + rolloutResult := progressivePerGroup(restClusterGroups, failureTimeout, existingClusterStatus) + return &strategy, rolloutResult, nil } -func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, length int, timeout time.Duration, statusFunc ClusterRolloutStatusFunc) RolloutResult { - rolloutClusters := map[string]ClusterRolloutStatus{} - timeoutClusters := map[string]ClusterRolloutStatus{} +func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, length int, timeout time.Duration, existingClusterStatus []ClusterRolloutStatus) RolloutResult { + rolloutClusters := []ClusterRolloutStatus{} + timeoutClusters := []ClusterRolloutStatus{} + existingClusters := make(map[string]bool) if length == 0 { return RolloutResult{ @@ -200,28 +214,45 @@ func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, len } } - clusters := clusterGroupsMap.GetClusters().UnsortedList() - clusterToGroupKey := clusterGroupsMap.ClusterToGroupKey() + for _, status := range existingClusterStatus { + if status.ClusterName == "" { + continue + } - // Sort the clusters in alphabetical order to ensure consistency. - sort.Strings(clusters) - for _, cluster := range clusters { - status := statusFunc(cluster) - if groupKey, exists := clusterToGroupKey[cluster]; exists { - status.GroupKey = groupKey + existingClusters[status.ClusterName] = true + if status.Status == Succeeded || status.Status == TimeOut { + continue } newStatus, needToRollout := determineRolloutStatusAndContinue(status, timeout) status.Status = newStatus.Status status.TimeOutTime = newStatus.TimeOutTime + if status.Status == TimeOut { + timeoutClusters = append(timeoutClusters, status) + continue + } if needToRollout { - rolloutClusters[cluster] = status + rolloutClusters = append(rolloutClusters, status) } - if status.Status == TimeOut { - timeoutClusters[cluster] = status + } + + clusters := clusterGroupsMap.GetClusters().UnsortedList() + clusterToGroupKey := clusterGroupsMap.ClusterToGroupKey() + // Sort the clusters in alphabetical order to ensure consistency. + sort.Strings(clusters) + for _, cluster := range clusters { + if existingClusters[cluster] { + continue } + status := ClusterRolloutStatus{ + ClusterName: cluster, + Status: ToApply, + GroupKey: clusterToGroupKey[cluster], + } + rolloutClusters = append(rolloutClusters, status) + if len(rolloutClusters)%length == 0 && len(rolloutClusters) > 0 { return RolloutResult{ ClustersToRollout: rolloutClusters, @@ -236,32 +267,54 @@ func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, len } } -func progressivePerGroup(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, timeout time.Duration, statusFunc ClusterRolloutStatusFunc) RolloutResult { - rolloutClusters := map[string]ClusterRolloutStatus{} - timeoutClusters := map[string]ClusterRolloutStatus{} +func progressivePerGroup(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, timeout time.Duration, existingClusterStatus []ClusterRolloutStatus) RolloutResult { + rolloutClusters := []ClusterRolloutStatus{} + timeoutClusters := []ClusterRolloutStatus{} + existingClusters := make(map[string]bool) - clusterGroupKeys := clusterGroupsMap.GetOrderedGroupKeys() + for _, status := range existingClusterStatus { + if status.ClusterName == "" { + continue + } + existingClusters[status.ClusterName] = true + if status.Status == Succeeded || status.Status == TimeOut { + continue + } + + newStatus, needToRollout := determineRolloutStatusAndContinue(status, timeout) + status.Status = newStatus.Status + status.TimeOutTime = newStatus.TimeOutTime + + if status.Status == TimeOut { + timeoutClusters = append(timeoutClusters, status) + continue + } + if needToRollout { + rolloutClusters = append(rolloutClusters, status) + } + } + + clusterGroupKeys := clusterGroupsMap.GetOrderedGroupKeys() for _, key := range clusterGroupKeys { if subclusters, ok := clusterGroupsMap[key]; ok { // Iterate through clusters in the group - for _, cluster := range subclusters.UnsortedList() { - status := statusFunc(cluster) - status.GroupKey = key - - newStatus, needToRollout := determineRolloutStatusAndContinue(status, timeout) - status.Status = newStatus.Status - status.TimeOutTime = newStatus.TimeOutTime - - if needToRollout { - rolloutClusters[cluster] = status + clusters := subclusters.UnsortedList() + sort.Strings(clusters) + for _, cluster := range clusters { + if existingClusters[cluster] { + continue } - if status.Status == TimeOut { - timeoutClusters[cluster] = status + + status := ClusterRolloutStatus{ + ClusterName: cluster, + Status: ToApply, + GroupKey: key, } + rolloutClusters = append(rolloutClusters, status) } - // Return if there are clusters to rollout + // As it is perGroup Return if there are clusters to rollout if len(rolloutClusters) > 0 { return RolloutResult{ ClustersToRollout: rolloutClusters, diff --git a/cluster/v1alpha1/helpers_test.go b/cluster/v1alpha1/helpers_test.go index badcb6b28..0dd10770d 100644 --- a/cluster/v1alpha1/helpers_test.go +++ b/cluster/v1alpha1/helpers_test.go @@ -28,100 +28,123 @@ type FakePlacementDecisionGetter struct { FakeDecisions []*clusterv1beta1.PlacementDecision } +// dummyWorkload states +const ( + valid string = "valid" + applying = "applying" + done = "done" + missing = "missing" +) + +type dummyWorkload struct { + ClusterGroup clusterv1beta1.GroupKey + ClusterName string + State string + LastTransitionTime *metav1.Time +} + +func dummyWorkloadClusterRolloutStatusFunc(clusterName string, workload dummyWorkload) (ClusterRolloutStatus, error) { + // workload obj should be used to determine the clusterRolloutStatus. + switch workload.State { + case valid: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: ToApply, LastTransitionTime: workload.LastTransitionTime}, nil + case applying: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: Progressing, LastTransitionTime: workload.LastTransitionTime}, nil + case done: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: Succeeded, LastTransitionTime: workload.LastTransitionTime}, nil + case missing: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: Failed, LastTransitionTime: workload.LastTransitionTime}, nil + default: + return ClusterRolloutStatus{GroupKey: workload.ClusterGroup, ClusterName: clusterName, Status: ToApply, LastTransitionTime: workload.LastTransitionTime}, nil + } +} + +type testCase struct { + name string + rolloutStrategy RolloutStrategy + existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] + clusterRolloutStatusFunc ClusterRolloutStatusFunc[dummyWorkload] // Using type dummy workload obj + expectRolloutStrategy *RolloutStrategy + existingWorkloads []dummyWorkload + expectRolloutClusters []ClusterRolloutStatus + expectTimeOutClusters []ClusterRolloutStatus +} + func (f *FakePlacementDecisionGetter) List(selector labels.Selector, namespace string) (ret []*clusterv1beta1.PlacementDecision, err error) { return f.FakeDecisions, nil } func TestGetRolloutCluster_All(t *testing.T) { - tests := []struct { - name string - rolloutStrategy RolloutStrategy - existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] - clusterRolloutStatusFunc ClusterRolloutStatusFunc - expectRolloutStrategy *RolloutStrategy - expectRolloutClusters map[string]ClusterRolloutStatus - expectTimeOutClusters map[string]ClusterRolloutStatus - }{ + tests := []testCase{ { - name: "test rollout all with timeout 90s", + name: "test rollout all with timeout 90s witout workload created", rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Progressing, LastTransitionTime: &fakeTime_60s}, - "cluster3": {Status: Succeeded, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] - }, - expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster6": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, - }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, - }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + existingWorkloads: []dummyWorkload{}, + expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{}, }, { - name: "test rollout all (default timeout None)", - rolloutStrategy: RolloutStrategy{Type: All}, + name: "test rollout all with timeout 90s", + rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] - }, - expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{""}}}, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, TimeOutTime: &fakeTimeMax}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, TimeOutTime: &fakeTimeMax}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, - }, - { - name: "test rollout all with timeout 0s", - rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"0s"}}}, - existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster4", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, + ClusterName: "cluster5", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, }, - expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"0s"}}}, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, TimeOutTime: &fakeTime}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, TimeOutTime: &fakeTime}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, }, } @@ -132,8 +155,14 @@ func TestGetRolloutCluster_All(t *testing.T) { fakeGetter := FakePlacementDecisionGetter{} tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) - rolloutHandler, _ := NewRolloutHandler(tracker) - actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, test.clusterRolloutStatusFunc) + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) if !reflect.DeepEqual(actualRolloutStrategy.All, test.expectRolloutStrategy.All) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) @@ -151,177 +180,192 @@ func TestGetRolloutCluster_All(t *testing.T) { } func TestGetRolloutCluster_Progressive(t *testing.T) { - tests := []struct { - name string - rolloutStrategy RolloutStrategy - existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] - clusterRolloutStatusFunc ClusterRolloutStatusFunc - expectRolloutStrategy *RolloutStrategy - expectRolloutClusters map[string]ClusterRolloutStatus - expectTimeOutClusters map[string]ClusterRolloutStatus - }{ + tests := []testCase{ { - name: "test progressive rollout with timeout 90s", + name: "test progressive rollout with timeout 90s witout workload created", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{"90s"}, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Progressing, LastTransitionTime: &fakeTime_60s}, - "cluster3": {Status: Succeeded, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{"90s"}, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - "cluster6": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, - }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + existingWorkloads: []dummyWorkload{}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, + expectTimeOutClusters: []ClusterRolloutStatus{}, }, { - name: "test progressive rollout with timeout None and MaxConcurrency 50%", + name: "test progressive rollout with timeout 90s", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{""}, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{""}, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(2), }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, TimeOutTime: &fakeTimeMax}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Failed, TimeOutTime: &fakeTimeMax}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive rollout with timeout 0s and MaxConcurrency 3", + name: "test progressive rollout with mandatroyDecisionGroup and timeout 90s ", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{"0s"}, - MaxConcurrency: intstr.FromInt(3), // Maximum 3 clusters concurrently + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(3), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: Progressing}, - "cluster3": {Status: Succeeded}, - "cluster4": {Status: Failed}, - "cluster5": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - Timeout: Timeout{"0s"}, - MaxConcurrency: intstr.FromInt(3), // Maximum 3 clusters concurrently + MandatoryDecisionGroups: MandatoryDecisionGroups{ + MandatoryDecisionGroups: []MandatoryDecisionGroup{ + {GroupName: "group1"}, + }, + }, + Timeout: Timeout{"90s"}, + MaxConcurrency: intstr.FromInt(3), + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, TimeOutTime: &fakeTime}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: TimeOut, TimeOutTime: &fakeTime}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, }, { - name: "test progressive rollout with mandatory decision groups", + name: "test progressive rollout with timeout None and MaxConcurrency 50%", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, - MaxConcurrency: intstr.FromString("50%"), + Timeout: Timeout{"None"}, + MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: ToApply}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, - MaxConcurrency: intstr.FromString("50%"), - Timeout: Timeout{""}, + Timeout: Timeout{"None"}, + MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTimeMax_120s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTimeMax_60s}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, + expectTimeOutClusters: []ClusterRolloutStatus{}, }, { - name: "test progressive rollout with mandatory decision groups Succeed", + name: "test progressive rollout with mandatory decision groups failed", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -330,24 +374,16 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromInt(2), + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Succeeded}, - "cluster2": {Status: Succeeded}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: Succeeded}, - "cluster5": {Status: ToApply}, - "cluster6": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -356,18 +392,33 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromInt(2), - Timeout: Timeout{""}, + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{"90s"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster3": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive rollout with mandatory decision groups failed", + name: "test progressive rollout with mandatory decision groups Succeed", rolloutStrategy: RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -376,24 +427,15 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters - Timeout: Timeout{"0s"}, + MaxConcurrency: intstr.FromInt(3), }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed}, - "cluster2": {Status: Failed}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: Progressive, Progressive: &RolloutProgressive{ @@ -402,15 +444,30 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { {GroupName: "group1"}, }, }, - MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters - Timeout: Timeout{"0s"}, + MaxConcurrency: intstr.FromInt(3), + Timeout: Timeout{""}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_120s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 2}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, + expectTimeOutClusters: []ClusterRolloutStatus{}, }, } @@ -422,8 +479,14 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { fakeGetter := FakePlacementDecisionGetter{} tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) - rolloutHandler, _ := NewRolloutHandler(tracker) - actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, test.clusterRolloutStatusFunc) + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) if !reflect.DeepEqual(actualRolloutStrategy.Progressive, test.expectRolloutStrategy.Progressive) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) @@ -441,17 +504,9 @@ func TestGetRolloutCluster_Progressive(t *testing.T) { } func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { - tests := []struct { - name string - rolloutStrategy RolloutStrategy - existingScheduledClusterGroups map[clusterv1beta1.GroupKey]sets.Set[string] - clusterRolloutStatusFunc ClusterRolloutStatusFunc - expectRolloutStrategy *RolloutStrategy - expectRolloutClusters map[string]ClusterRolloutStatus - expectTimeOutClusters map[string]ClusterRolloutStatus - }{ + tests := []testCase{ { - name: "test progressive per group rollout with timeout 90s", + name: "test progressivePerGroup rollout with timeout 90s witout workload created", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -459,149 +514,166 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster3": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ Timeout: Timeout{"90s"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, - }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + existingWorkloads: []dummyWorkload{}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster1", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, }, + expectTimeOutClusters: []ClusterRolloutStatus{}, }, { - name: "test progressive per group rollout with timeout None", + name: "test progressivePerGroup rollout with timeout 90s", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster3": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - Timeout: Timeout{""}, + Timeout: Timeout{"90s"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTimeMax_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTimeMax_120s}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive per group rollout with timeout 0s", + name: "test progressivePerGroup rollout with timeout 90s and first group timeOut", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed, LastTransitionTime: &fakeTime_60s}, - "cluster2": {Status: Failed, LastTransitionTime: &fakeTime_120s}, - "cluster3": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster3": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster4": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply, LastTransitionTime: &fakeTime_60s}, - "cluster6": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime_60s}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_120s}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, }, { - name: "test progressive per group rollout with mandatory decision groups", + name: "test progressivePerGroup rollout with timeout None and first group failing", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, + Timeout: Timeout{"None"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: ToApply}, - "cluster2": {Status: ToApply}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ - MandatoryDecisionGroups: MandatoryDecisionGroups{ - MandatoryDecisionGroups: []MandatoryDecisionGroup{ - {GroupName: "group1"}, - }, - }, + Timeout: Timeout{"None"}, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, + }, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTimeMax_120s}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{}, }, { - name: "test progressive per group rollout with mandatory decision groups Succeed", + name: "test ProgressivePerGroup rollout with mandatroyDecisionGroup failing and timeout 90s ", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -610,23 +682,14 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {GroupName: "group1"}, }, }, + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4"), - {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Succeeded}, - "cluster2": {Status: Succeeded}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: Succeeded}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -635,15 +698,37 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {GroupName: "group1"}, }, }, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: missing, + LastTransitionTime: &fakeTime_120s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: missing, + LastTransitionTime: &fakeTime_120s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster3": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}, Status: ToApply}, + expectRolloutClusters: []ClusterRolloutStatus{}, + expectTimeOutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster2", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: TimeOut, LastTransitionTime: &fakeTime_120s, TimeOutTime: &fakeTime_30s}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, }, { - name: "test progressive per group rollout with mandatory decision groups failed", + name: "test ProgressivePerGroup rollout with mandatroyDecisionGroup Succeeded and timeout 90s ", rolloutStrategy: RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -652,23 +737,15 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {GroupName: "group1"}, }, }, - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, }, }, existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ - {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"), - {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"), - }, - clusterRolloutStatusFunc: func(clusterName string) ClusterRolloutStatus { - clustersRolloutStatus := map[string]ClusterRolloutStatus{ - "cluster1": {Status: Failed}, - "cluster2": {Status: Failed}, - "cluster3": {Status: ToApply}, - "cluster4": {Status: ToApply}, - "cluster5": {Status: ToApply}, - } - return clustersRolloutStatus[clusterName] + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + {GroupName: "", GroupIndex: 2}: sets.New[string]("cluster7", "cluster8", "cluster9"), }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, expectRolloutStrategy: &RolloutStrategy{ Type: ProgressivePerGroup, ProgressivePerGroup: &RolloutProgressivePerGroup{ @@ -677,14 +754,35 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { {GroupName: "group1"}, }, }, - Timeout: Timeout{"0s"}, + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster3", + State: done, + LastTransitionTime: &fakeTime_60s, }, }, - expectRolloutClusters: map[string]ClusterRolloutStatus{ - "cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, - "cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Failed, TimeOutTime: &fakeTimeMax}, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, + {ClusterName: "cluster6", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: ToApply}, }, - expectTimeOutClusters: map[string]ClusterRolloutStatus{}, + expectTimeOutClusters: []ClusterRolloutStatus{}, }, } @@ -696,8 +794,14 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { fakeGetter := FakePlacementDecisionGetter{} tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) - rolloutHandler, _ := NewRolloutHandler(tracker) - actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, test.clusterRolloutStatusFunc) + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) if !reflect.DeepEqual(actualRolloutStrategy.ProgressivePerGroup, test.expectRolloutStrategy.ProgressivePerGroup) { t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) @@ -714,6 +818,90 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) { } } +func TestGetRolloutCluster_ClusterAdded(t *testing.T) { + test := testCase{ + name: "test progressivePerGroup rollout with timeout 90s and cluster added after rollout start.", + rolloutStrategy: RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{ + {GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2", "cluster3"), + {GroupName: "", GroupIndex: 1}: sets.New[string]("cluster4", "cluster5", "cluster6"), + }, + clusterRolloutStatusFunc: dummyWorkloadClusterRolloutStatusFunc, + expectRolloutStrategy: &RolloutStrategy{ + Type: ProgressivePerGroup, + ProgressivePerGroup: &RolloutProgressivePerGroup{ + Timeout: Timeout{"90s"}, + }, + }, + existingWorkloads: []dummyWorkload{ + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster1", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, + ClusterName: "cluster2", + State: done, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster4", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + { + ClusterGroup: clusterv1beta1.GroupKey{GroupIndex: 1}, + ClusterName: "cluster5", + State: applying, + LastTransitionTime: &fakeTime_60s, + }, + }, + expectRolloutClusters: []ClusterRolloutStatus{ + {ClusterName: "cluster4", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster5", GroupKey: clusterv1beta1.GroupKey{GroupIndex: 1}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s}, + {ClusterName: "cluster3", GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply}, + }, + expectTimeOutClusters: []ClusterRolloutStatus{}, + } + + // Set the fake time for testing + RolloutClock = testingclock.NewFakeClock(fakeTime.Time) + + // Init fake placement decision tracker + fakeGetter := FakePlacementDecisionGetter{} + tracker := clusterv1beta1.NewPlacementDecisionClustersTrackerWithGroups(nil, &fakeGetter, test.existingScheduledClusterGroups) + + rolloutHandler, _ := NewRolloutHandler(tracker, test.clusterRolloutStatusFunc) + existingRolloutClusters := []ClusterRolloutStatus{} + for _, workload := range test.existingWorkloads { + clsRolloutStatus, _ := test.clusterRolloutStatusFunc(workload.ClusterName, workload) + existingRolloutClusters = append(existingRolloutClusters, clsRolloutStatus) + } + + actualRolloutStrategy, actualRolloutResult, _ := rolloutHandler.GetRolloutCluster(test.rolloutStrategy, existingRolloutClusters) + + if !reflect.DeepEqual(actualRolloutStrategy.ProgressivePerGroup, test.expectRolloutStrategy.ProgressivePerGroup) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect strategy : %v, actual : %v", test.name, test.expectRolloutStrategy, actualRolloutStrategy) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersToRollout, test.expectRolloutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect rollout clusters: %v, actual : %v", test.name, test.expectRolloutClusters, actualRolloutResult.ClustersToRollout) + return + } + if !reflect.DeepEqual(actualRolloutResult.ClustersTimeOut, test.expectTimeOutClusters) { + t.Errorf("Case: %v, Failed to run NewRolloutHandler. Expect timeout clusters: %v, actual : %v", test.name, test.expectTimeOutClusters, actualRolloutResult.ClustersTimeOut) + return + } +} + func TestNeedToUpdate(t *testing.T) { testCases := []struct { name string diff --git a/cluster/v1alpha1/zz_generated.deepcopy.go b/cluster/v1alpha1/zz_generated.deepcopy.go index a72d4c816..00938a885 100644 --- a/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/cluster/v1alpha1/zz_generated.deepcopy.go @@ -315,14 +315,14 @@ func (in *RolloutResult) DeepCopyInto(out *RolloutResult) { *out = *in if in.ClustersToRollout != nil { in, out := &in.ClustersToRollout, &out.ClustersToRollout - *out = make(map[string]ClusterRolloutStatus, len(*in)) + *out = make([]ClusterRolloutStatus, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() } } if in.ClustersTimeOut != nil { in, out := &in.ClustersTimeOut, &out.ClustersTimeOut - *out = make(map[string]ClusterRolloutStatus, len(*in)) + *out = make([]ClusterRolloutStatus, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() }