From 63c0a42f9a715537a03a6d8cb3164cafb5edad98 Mon Sep 17 00:00:00 2001 From: "jinle.xjl" Date: Mon, 25 Sep 2023 20:09:58 +0800 Subject: [PATCH 01/12] fix --- .../controller/moduledeployment_controller.go | 41 ++++++++----------- .../controller/modulereplicaset_controller.go | 3 +- .../internal/controller/pod_controller.go | 8 ++-- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index 788b26520..ede1a1c90 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -307,11 +307,21 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, batchCount = moduleDeployment.Spec.OperationStrategy.BatchCount curBatch = moduleDeployment.Status.ReleaseStatus.CurrentBatch - curReplicas = newRS.Status.Replicas - expReplicas = moduleDeployment.Spec.Replicas - deltaReplicas = expReplicas - newRS.Spec.Replicas + curReplicas = newRS.Status.Replicas + expReplicas = moduleDeployment.Spec.Replicas ) - if deltaReplicas == 0 { + + if batchCount <= 0 { + batchCount = 1 + } + + // wait moduleReplicaset ready + if replicas := (curBatch - 1) * (moduleDeployment.Spec.Replicas / batchCount); replicas > curReplicas { + log.Log.Info(fmt.Sprintf("newRs is not ready, expect replicas %v, but got %v", replicas, curReplicas)) + return true, nil + } + + if curReplicas == expReplicas { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted moduleDeployment.Status.ReleaseStatus.LastTransitionTime = metav1.Now() moduleDeployment.Status.Conditions = append(moduleDeployment.Status.Conditions, moduledeploymentv1alpha1.ModuleDeploymentCondition{ @@ -323,28 +333,9 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, return false, r.Status().Update(ctx, moduleDeployment) } - if expReplicas < batchCount { - batchCount = expReplicas - } - - if batchCount <= 0 { - batchCount = 1 - } - - // wait moduleReplicaset ready - if newRS.Spec.Replicas != curReplicas { - log.Log.Info(fmt.Sprintf("newRs is not ready, expect replicas %v, but got %v", newRS.Spec.Replicas, curReplicas)) - return true, nil - } - - replicas := int32(0) - // use beta strategy - if batchCount != 1 && curBatch == 1 && moduleDeployment.Spec.OperationStrategy.UseBeta { - replicas = 1 - } else if curBatch == batchCount { // if it's the last batch + replicas := curBatch * (moduleDeployment.Spec.Replicas / batchCount) + if curBatch == batchCount { // is the last batch replicas = expReplicas - } else { - replicas = newRS.Spec.Replicas + (curBatch)*int32(math.Floor(float64(deltaReplicas)/float64(batchCount)+0.5)) } err := r.updateModuleReplicas(ctx, replicas, moduleDeployment, newRS) diff --git a/module-controller/internal/controller/modulereplicaset_controller.go b/module-controller/internal/controller/modulereplicaset_controller.go index 9e05e518c..66ac4d657 100644 --- a/module-controller/internal/controller/modulereplicaset_controller.go +++ b/module-controller/internal/controller/modulereplicaset_controller.go @@ -19,10 +19,11 @@ package controller import ( "context" "fmt" - "k8s.io/apimachinery/pkg/selection" "sort" "strconv" + "k8s.io/apimachinery/pkg/selection" + "golang.org/x/tools/container/intsets" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/module-controller/internal/controller/pod_controller.go b/module-controller/internal/controller/pod_controller.go index deacb038b..ee06f9347 100644 --- a/module-controller/internal/controller/pod_controller.go +++ b/module-controller/internal/controller/pod_controller.go @@ -18,14 +18,16 @@ package controller import ( "context" - moduledeploymentv1alpha1 "github.com/sofastack/sofa-serverless/api/v1alpha1" - "github.com/sofastack/sofa-serverless/internal/constants/label" - "github.com/sofastack/sofa-serverless/internal/utils" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/reconcile" + moduledeploymentv1alpha1 "github.com/sofastack/sofa-serverless/api/v1alpha1" + "github.com/sofastack/sofa-serverless/internal/constants/label" + "github.com/sofastack/sofa-serverless/internal/utils" + "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" From 5783db70ba6ee36f985939d944f1203e6f72f394 Mon Sep 17 00:00:00 2001 From: charlie Date: Tue, 26 Sep 2023 22:34:00 +0800 Subject: [PATCH 02/12] fix: fix batch deploy when scaling up or scaling down --- .../controller/moduledeployment_controller.go | 41 +++++++++++-------- .../internal/controller/suite_test.go | 1 + 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index ede1a1c90..788b26520 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -307,21 +307,11 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, batchCount = moduleDeployment.Spec.OperationStrategy.BatchCount curBatch = moduleDeployment.Status.ReleaseStatus.CurrentBatch - curReplicas = newRS.Status.Replicas - expReplicas = moduleDeployment.Spec.Replicas + curReplicas = newRS.Status.Replicas + expReplicas = moduleDeployment.Spec.Replicas + deltaReplicas = expReplicas - newRS.Spec.Replicas ) - - if batchCount <= 0 { - batchCount = 1 - } - - // wait moduleReplicaset ready - if replicas := (curBatch - 1) * (moduleDeployment.Spec.Replicas / batchCount); replicas > curReplicas { - log.Log.Info(fmt.Sprintf("newRs is not ready, expect replicas %v, but got %v", replicas, curReplicas)) - return true, nil - } - - if curReplicas == expReplicas { + if deltaReplicas == 0 { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted moduleDeployment.Status.ReleaseStatus.LastTransitionTime = metav1.Now() moduleDeployment.Status.Conditions = append(moduleDeployment.Status.Conditions, moduledeploymentv1alpha1.ModuleDeploymentCondition{ @@ -333,9 +323,28 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, return false, r.Status().Update(ctx, moduleDeployment) } - replicas := curBatch * (moduleDeployment.Spec.Replicas / batchCount) - if curBatch == batchCount { // is the last batch + if expReplicas < batchCount { + batchCount = expReplicas + } + + if batchCount <= 0 { + batchCount = 1 + } + + // wait moduleReplicaset ready + if newRS.Spec.Replicas != curReplicas { + log.Log.Info(fmt.Sprintf("newRs is not ready, expect replicas %v, but got %v", newRS.Spec.Replicas, curReplicas)) + return true, nil + } + + replicas := int32(0) + // use beta strategy + if batchCount != 1 && curBatch == 1 && moduleDeployment.Spec.OperationStrategy.UseBeta { + replicas = 1 + } else if curBatch == batchCount { // if it's the last batch replicas = expReplicas + } else { + replicas = newRS.Spec.Replicas + (curBatch)*int32(math.Floor(float64(deltaReplicas)/float64(batchCount)+0.5)) } err := r.updateModuleReplicas(ctx, replicas, moduleDeployment, newRS) diff --git a/module-controller/internal/controller/suite_test.go b/module-controller/internal/controller/suite_test.go index 5f069c76f..030cf9032 100644 --- a/module-controller/internal/controller/suite_test.go +++ b/module-controller/internal/controller/suite_test.go @@ -18,6 +18,7 @@ package controller import ( "fmt" + "github.com/sofastack/sofa-serverless/internal/arklet" "os" "path/filepath" "testing" From 16b912445776bb8213bfa5fda1de0a0a41143b60 Mon Sep 17 00:00:00 2001 From: charlie Date: Wed, 27 Sep 2023 12:06:38 +0800 Subject: [PATCH 03/12] fix: module_deployment_controller unit test --- module-controller/internal/controller/suite_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/module-controller/internal/controller/suite_test.go b/module-controller/internal/controller/suite_test.go index 030cf9032..5f069c76f 100644 --- a/module-controller/internal/controller/suite_test.go +++ b/module-controller/internal/controller/suite_test.go @@ -18,7 +18,6 @@ package controller import ( "fmt" - "github.com/sofastack/sofa-serverless/internal/arklet" "os" "path/filepath" "testing" From 305299fa4daf8fa7364cf69a4666b32e57519946 Mon Sep 17 00:00:00 2001 From: charlie Date: Sat, 19 Aug 2023 15:28:25 +0800 Subject: [PATCH 04/12] feat: support batch deploy --- .../controller/moduledeployment_controller.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index 788b26520..b1dc8c5df 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -23,17 +23,11 @@ import ( "strconv" "time" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - - moduledeploymentv1alpha1 "github.com/sofastack/sofa-serverless/api/v1alpha1" - "github.com/sofastack/sofa-serverless/internal/constants/finalizer" - "github.com/sofastack/sofa-serverless/internal/constants/label" - "github.com/sofastack/sofa-serverless/internal/utils" - v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" @@ -41,6 +35,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + moduledeploymentv1alpha1 "github.com/sofastack/sofa-serverless/api/v1alpha1" + "github.com/sofastack/sofa-serverless/internal/constants/finalizer" + "github.com/sofastack/sofa-serverless/internal/constants/label" + "github.com/sofastack/sofa-serverless/internal/utils" ) // ModuleDeploymentReconciler reconciles a ModuleDeployment object From c39761b4100b08d36f969f0d88c44259467cf850 Mon Sep 17 00:00:00 2001 From: charlie Date: Sun, 20 Aug 2023 14:57:20 +0800 Subject: [PATCH 05/12] feat: support batchConfirm && GrayTime Signed-off-by: charlie --- module-controller/cmd/main.go | 4 +- .../controller/moduledeployment_controller.go | 75 ++++++++++--------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/module-controller/cmd/main.go b/module-controller/cmd/main.go index 6165fa05b..ef8efe437 100644 --- a/module-controller/cmd/main.go +++ b/module-controller/cmd/main.go @@ -18,9 +18,10 @@ package main import ( "flag" - "go.uber.org/zap/zapcore" "os" + "go.uber.org/zap/zapcore" + "github.com/sofastack/sofa-serverless/internal/controller" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -30,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index b1dc8c5df..0015345f3 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -19,7 +19,6 @@ package controller import ( "context" "fmt" - "math" "strconv" "time" @@ -110,15 +109,7 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting: - // update moduleReplicaSet - enqueue, err := r.updateModuleReplicaSet(ctx, moduleDeployment, newRS) - if err != nil { - return ctrl.Result{}, err - } - if enqueue { - requeueAfter := utils.GetNextReconcileTime(time.Now()) - return ctrl.Result{RequeueAfter: requeueAfter}, nil - } + return r.updateModuleReplicaSet(moduleDeployment, newRS, oldRSs) case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted: if moduleDeployment.Spec.Replicas != newRS.Spec.Replicas { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting @@ -127,6 +118,21 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } } + case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation: + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused + if err := r.Status().Update(ctx, moduleDeployment); err != nil { + return ctrl.Result{}, err + } + case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused: + moduleDeployment.Spec.Pause = true + if err := r.Update(ctx, moduleDeployment); err != nil { + return ctrl.Result{}, err + } + + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + if err := r.Status().Update(ctx, moduleDeployment); err != nil { + return ctrl.Result{}, err + } if !moduleVersionChanged && isUrlChange(moduleDeployment.Spec.Template.Spec.Module, newRS.Spec.Template.Spec.Module) { newRS.Spec.Template.Spec.Module = moduleDeployment.Spec.Template.Spec.Module @@ -300,9 +306,11 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicas( return nil } -func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, moduleDeployment *moduledeploymentv1alpha1.ModuleDeployment, - newRS *moduledeploymentv1alpha1.ModuleReplicaSet) (bool, error) { +func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *moduledeploymentv1alpha1.ModuleDeployment, + newRS *moduledeploymentv1alpha1.ModuleReplicaSet, oldRSs []*moduledeploymentv1alpha1.ModuleReplicaSet) (ctrl.Result, error) { var ( + ctx = context.TODO() + batchCount = moduleDeployment.Spec.OperationStrategy.BatchCount curBatch = moduleDeployment.Status.ReleaseStatus.CurrentBatch @@ -319,28 +327,11 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, LastTransitionTime: metav1.Now(), Message: "deployment release progress completed", }) - return false, r.Status().Update(ctx, moduleDeployment) + return ctrl.Result{}, r.Status().Update(ctx, moduleDeployment) } - if expReplicas < batchCount { - batchCount = expReplicas - } - - if batchCount <= 0 { - batchCount = 1 - } - - // wait moduleReplicaset ready - if newRS.Spec.Replicas != curReplicas { - log.Log.Info(fmt.Sprintf("newRs is not ready, expect replicas %v, but got %v", newRS.Spec.Replicas, curReplicas)) - return true, nil - } - - replicas := int32(0) - // use beta strategy - if batchCount != 1 && curBatch == 1 && moduleDeployment.Spec.OperationStrategy.UseBeta { - replicas = 1 - } else if curBatch == batchCount { // if it's the last batch + replicas := curBatch * (moduleDeployment.Spec.Replicas / batchCount) + if curBatch == batchCount { // is the last batch replicas = expReplicas } else { replicas = newRS.Spec.Replicas + (curBatch)*int32(math.Floor(float64(deltaReplicas)/float64(batchCount)+0.5)) @@ -348,12 +339,26 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, err := r.updateModuleReplicas(ctx, replicas, moduleDeployment, newRS) if err != nil { - return false, err + return ctrl.Result{}, err } moduleDeployment.Status.ReleaseStatus.CurrentBatch += 1 - moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting moduleDeployment.Status.ReleaseStatus.LastTransitionTime = metav1.Now() + + var grayTime = 0 + + if moduleDeployment.Spec.OperationStrategy.NeedConfirm { + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation + } else if grayTime = int(moduleDeployment.Spec.OperationStrategy.GrayTimeBetweenBatchSeconds); grayTime != 0 { + if curBatch == batchCount { + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + } else { + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused + } + } else { + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + } + moduleDeployment.Status.Conditions = append(moduleDeployment.Status.Conditions, moduledeploymentv1alpha1.ModuleDeploymentCondition{ Type: moduledeploymentv1alpha1.DeploymentProgressing, Status: corev1.ConditionTrue, @@ -361,7 +366,7 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, Message: fmt.Sprintf("deployment release: curbatch %v, batchCount %v", curBatch, batchCount), }) - return false, r.Status().Update(ctx, moduleDeployment) + return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(grayTime) * time.Second}, r.Status().Update(ctx, moduleDeployment) } // generate module replicas From 3607e6fa5c64b6034726963b9c96525f83d61e31 Mon Sep 17 00:00:00 2001 From: charlie Date: Mon, 25 Sep 2023 19:39:35 +0800 Subject: [PATCH 06/12] feat: add useBeta strategy --- module-controller/Makefile | 2 +- .../controller/module_controller_suit_test.go | 11 ++- .../controller/moduledeployment_controller.go | 61 ++++++++---- .../moduledeployment_controller_suit_test.go | 96 ++++++++++++++++++- 4 files changed, 144 insertions(+), 26 deletions(-) diff --git a/module-controller/Makefile b/module-controller/Makefile index d7a66b03e..0ba88d6c1 100644 --- a/module-controller/Makefile +++ b/module-controller/Makefile @@ -62,7 +62,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out + test=true KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out ##@ Build diff --git a/module-controller/internal/controller/module_controller_suit_test.go b/module-controller/internal/controller/module_controller_suit_test.go index b761d40a4..1aeff4025 100644 --- a/module-controller/internal/controller/module_controller_suit_test.go +++ b/module-controller/internal/controller/module_controller_suit_test.go @@ -2,19 +2,22 @@ package controller import ( "context" + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "github.com/sofastack/sofa-serverless/api/v1alpha1" "github.com/sofastack/sofa-serverless/internal/constants/finalizer" "github.com/sofastack/sofa-serverless/internal/constants/label" "github.com/sofastack/sofa-serverless/internal/utils" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - "time" ) var _ = Describe("Module Controller", func() { diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index 0015345f3..c22508004 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "math" "strconv" "time" @@ -119,20 +120,22 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req } } case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation: - moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused - if err := r.Status().Update(ctx, moduleDeployment); err != nil { - return ctrl.Result{}, err - } - case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused: moduleDeployment.Spec.Pause = true if err := r.Update(ctx, moduleDeployment); err != nil { return ctrl.Result{}, err } - moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused if err := r.Status().Update(ctx, moduleDeployment); err != nil { return ctrl.Result{}, err } + case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused: + if !moduleDeployment.Spec.Pause { + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + if err := r.Status().Update(ctx, moduleDeployment); err != nil { + return ctrl.Result{}, err + } + } if !moduleVersionChanged && isUrlChange(moduleDeployment.Spec.Template.Spec.Module, newRS.Spec.Template.Spec.Module) { newRS.Spec.Template.Spec.Module = moduleDeployment.Spec.Template.Spec.Module @@ -318,6 +321,7 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo expReplicas = moduleDeployment.Spec.Replicas deltaReplicas = expReplicas - newRS.Spec.Replicas ) + if deltaReplicas == 0 { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted moduleDeployment.Status.ReleaseStatus.LastTransitionTime = metav1.Now() @@ -330,8 +334,25 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo return ctrl.Result{}, r.Status().Update(ctx, moduleDeployment) } - replicas := curBatch * (moduleDeployment.Spec.Replicas / batchCount) - if curBatch == batchCount { // is the last batch + if expReplicas < batchCount { + batchCount = expReplicas + } + + if batchCount <= 0 { + batchCount = 1 + } + + // wait moduleReplicaset ready + if newRS.Spec.Replicas != curReplicas { + log.Log.Info(fmt.Sprintf("newRs is not ready, expect replicas %v, but got %v", newRS.Spec.Replicas, curReplicas)) + return ctrl.Result{Requeue: true, RequeueAfter: utils.GetNextReconcileTime(time.Now())}, nil + } + + replicas := int32(0) + // use beta strategy + if batchCount != 1 && curBatch == 1 && moduleDeployment.Spec.OperationStrategy.UseBeta { + replicas = 1 + } else if curBatch == batchCount { // if it's the last batch replicas = expReplicas } else { replicas = newRS.Spec.Replicas + (curBatch)*int32(math.Floor(float64(deltaReplicas)/float64(batchCount)+0.5)) @@ -344,19 +365,19 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo moduleDeployment.Status.ReleaseStatus.CurrentBatch += 1 moduleDeployment.Status.ReleaseStatus.LastTransitionTime = metav1.Now() - - var grayTime = 0 - - if moduleDeployment.Spec.OperationStrategy.NeedConfirm { - moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation - } else if grayTime = int(moduleDeployment.Spec.OperationStrategy.GrayTimeBetweenBatchSeconds); grayTime != 0 { - if curBatch == batchCount { - moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting - } else { - moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + + var grayTime int + if curBatch != batchCount { + if moduleDeployment.Spec.OperationStrategy.NeedConfirm { // use NeedConfirm Strategy + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation + } else if grayTime = int(moduleDeployment.Spec.OperationStrategy.GrayTimeBetweenBatchSeconds); grayTime != 0 { + if curBatch == batchCount { + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + } else { + moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused + } } - } else { - moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting } moduleDeployment.Status.Conditions = append(moduleDeployment.Status.Conditions, moduledeploymentv1alpha1.ModuleDeploymentCondition{ diff --git a/module-controller/internal/controller/moduledeployment_controller_suit_test.go b/module-controller/internal/controller/moduledeployment_controller_suit_test.go index 313c464f1..1562dbb01 100644 --- a/module-controller/internal/controller/moduledeployment_controller_suit_test.go +++ b/module-controller/internal/controller/moduledeployment_controller_suit_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -147,7 +148,9 @@ var _ = Describe("ModuleDeployment Controller", func() { var newModuleDeployment v1alpha1.ModuleDeployment Expect(k8sClient.Get(context.TODO(), key, &newModuleDeployment)).Should(Succeed()) newModuleDeployment.Spec.Replicas += 1 - Expect(k8sClient.Update(context.TODO(), &newModuleDeployment)).Should(Succeed()) + Eventually(func() bool { + return k8sClient.Update(context.TODO(), &newModuleDeployment) == nil + }, timeout, interval).Should(BeTrue()) Eventually(func() bool { set := map[string]string{ @@ -189,6 +192,97 @@ var _ = Describe("ModuleDeployment Controller", func() { }) }) + Context("test batchConfirm strategy", func() { + namespace := "default" + moduleDeploymentName := "module-deployment-test-for-batch-confirm" + nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} + moduleDeployment := prepareModuleDeployment(namespace, moduleDeploymentName) + moduleDeployment.Spec.Replicas = 2 + moduleDeployment.Spec.OperationStrategy.NeedConfirm = true + moduleDeployment.Spec.OperationStrategy.BatchCount = 2 + + It("0. prepare 2 pods", func() { + Eventually(func() bool { + pod := preparePod("fake-pod-3") + pod.Labels[fmt.Sprintf("%s-%s", label.ModuleNameLabel, "dynamic-provider")] = "1.0.0" + if err := k8sClient.Create(context.TODO(), &pod); err != nil { + return false + } + // when install module, the podIP is necessary + pod.Status.PodIP = "127.0.0.1" + return k8sClient.Status().Update(context.TODO(), &pod) == nil + }, timeout, interval).Should(BeTrue()) + + }) + + It("1. create a new moduleDeployment", func() { + Expect(k8sClient.Create(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + + It("2. check if the replicas is 1", func() { + // todo: we just check deployment.status.replicas rather than modulereplicaset + Eventually(func() bool { + if err := k8sClient.Get(context.TODO(), nn, &moduleDeployment); err != nil { + return false + } + + if !moduleDeployment.Spec.Pause { + return false + } + + set := map[string]string{ + label.ModuleDeploymentLabel: moduleDeployment.Name, + } + replicaSetList := &moduledeploymentv1alpha1.ModuleReplicaSetList{} + err := k8sClient.List(context.TODO(), replicaSetList, &client.ListOptions{LabelSelector: labels.SelectorFromSet(set)}, client.InNamespace(moduleDeployment.Namespace)) + if err != nil || len(replicaSetList.Items) == 0 { + return false + } + + maxVersion := 0 + var newRS *moduledeploymentv1alpha1.ModuleReplicaSet + for i := 0; i < len(replicaSetList.Items); i++ { + version, err := getRevision(&replicaSetList.Items[i]) + if err != nil { + return false + } + if version > maxVersion { + maxVersion = version + newRS = &replicaSetList.Items[i] + } + } + + // the replicas of new replicaset must be equal to newModuleDeployment + return newRS != nil && + newRS.Status.Replicas == newRS.Spec.Replicas && + newRS.Status.Replicas == 1 + }, timeout, interval).Should(BeTrue()) + }) + + It("3. resume", func() { + Eventually(func() bool { + Expect(k8sClient.Get(context.TODO(), nn, &moduleDeployment)).Should(Succeed()) + + moduleDeployment.Spec.Pause = false + return Expect(k8sClient.Update(context.TODO(), &moduleDeployment)).Should(Succeed()) + }, timeout, interval).Should(BeTrue()) + }) + + It("4. check if the moduleDeployment status is completed", func() { + Eventually(func() bool { + if k8sClient.Get(context.TODO(), nn, &moduleDeployment) != nil { + return false + } + + if moduleDeployment.Spec.Pause != false { + return false + } + + return moduleDeployment.Status.ReleaseStatus.Progress == moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted + }, timeout, interval).Should(BeTrue()) + }) + }) + Context("delete module deployment", func() { It("clean module replicaset and deployment", func() { key := types.NamespacedName{ From f94ef64e2ce2282de0b4294dbd4ea59430353fb6 Mon Sep 17 00:00:00 2001 From: charlie Date: Wed, 27 Sep 2023 16:14:56 +0800 Subject: [PATCH 07/12] fix: grayTime strategy --- .../api/v1alpha1/moduledeployment_types.go | 2 + .../api/v1alpha1/zz_generated.deepcopy.go | 1 + ...rverless.alipay.com_moduledeployments.yaml | 3 + ...nt_v1alpha1_moduledeployment_provider.yaml | 3 +- .../controller/moduledeployment_controller.go | 6 +- .../moduledeployment_controller_suit_test.go | 101 +++++++----------- 6 files changed, 49 insertions(+), 67 deletions(-) diff --git a/module-controller/api/v1alpha1/moduledeployment_types.go b/module-controller/api/v1alpha1/moduledeployment_types.go index 95f47e6e1..cf78ae296 100644 --- a/module-controller/api/v1alpha1/moduledeployment_types.go +++ b/module-controller/api/v1alpha1/moduledeployment_types.go @@ -81,6 +81,8 @@ type ReleaseStatus struct { // Last time the release transitioned from one status to another. LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + NextReconcileTime metav1.Time `json:"nextReconcileTime,omitempty"` } type ModuleDeploymentCondition struct { diff --git a/module-controller/api/v1alpha1/zz_generated.deepcopy.go b/module-controller/api/v1alpha1/zz_generated.deepcopy.go index 3fcfe8b62..0be76c3d3 100644 --- a/module-controller/api/v1alpha1/zz_generated.deepcopy.go +++ b/module-controller/api/v1alpha1/zz_generated.deepcopy.go @@ -437,6 +437,7 @@ func (in *ModuleTemplateSpec) DeepCopy() *ModuleTemplateSpec { func (in *ReleaseStatus) DeepCopyInto(out *ReleaseStatus) { *out = *in in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + in.NextReconcileTime.DeepCopyInto(&out.NextReconcileTime) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReleaseStatus. diff --git a/module-controller/config/crd/bases/serverless.alipay.com_moduledeployments.yaml b/module-controller/config/crd/bases/serverless.alipay.com_moduledeployments.yaml index bce62898c..8bac35f67 100644 --- a/module-controller/config/crd/bases/serverless.alipay.com_moduledeployments.yaml +++ b/module-controller/config/crd/bases/serverless.alipay.com_moduledeployments.yaml @@ -220,6 +220,9 @@ spec: to another. format: date-time type: string + nextReconcileTime: + format: date-time + type: string progress: description: The phase current release reach type: string diff --git a/module-controller/config/samples/module-deployment_v1alpha1_moduledeployment_provider.yaml b/module-controller/config/samples/module-deployment_v1alpha1_moduledeployment_provider.yaml index 2ae5c59d4..9c4cf9a4d 100644 --- a/module-controller/config/samples/module-deployment_v1alpha1_moduledeployment_provider.yaml +++ b/module-controller/config/samples/module-deployment_v1alpha1_moduledeployment_provider.yaml @@ -18,7 +18,8 @@ spec: url: http://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/dynamic-provider-1.0.0-ark-biz.jar replicas: 1 operationStrategy: - needConfirm: true + needConfirm: false + grayTimeBetweenBatchSeconds: 120 useBeta: false batchCount: 1 schedulingStrategy: diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index c22508004..bbd399a2f 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -130,7 +130,7 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused: - if !moduleDeployment.Spec.Pause { + if !moduleDeployment.Spec.Pause && time.Since(moduleDeployment.Status.ReleaseStatus.NextReconcileTime.Time) >= 0 { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting if err := r.Status().Update(ctx, moduleDeployment); err != nil { return ctrl.Result{}, err @@ -363,8 +363,9 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo return ctrl.Result{}, err } + now := metav1.Now() moduleDeployment.Status.ReleaseStatus.CurrentBatch += 1 - moduleDeployment.Status.ReleaseStatus.LastTransitionTime = metav1.Now() + moduleDeployment.Status.ReleaseStatus.LastTransitionTime = now moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting var grayTime int @@ -375,6 +376,7 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo if curBatch == batchCount { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting } else { + moduleDeployment.Status.ReleaseStatus.NextReconcileTime = metav1.NewTime(now.Add(time.Duration(grayTime) * time.Second)) moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressPaused } } diff --git a/module-controller/internal/controller/moduledeployment_controller_suit_test.go b/module-controller/internal/controller/moduledeployment_controller_suit_test.go index 1562dbb01..a17f93d93 100644 --- a/module-controller/internal/controller/moduledeployment_controller_suit_test.go +++ b/module-controller/internal/controller/moduledeployment_controller_suit_test.go @@ -99,42 +99,9 @@ var _ = Describe("ModuleDeployment Controller", func() { Expect(k8sClient.Update(context.TODO(), &newModuleDeployment)).Should(Succeed()) Eventually(func() bool { - set := map[string]string{ - label.ModuleDeploymentLabel: moduleDeployment.Name, - } - replicaSetList := &moduledeploymentv1alpha1.ModuleReplicaSetList{} - err := k8sClient.List(context.TODO(), replicaSetList, &client.ListOptions{LabelSelector: labels.SelectorFromSet(set)}, client.InNamespace(moduleDeployment.Namespace)) - if err != nil || len(replicaSetList.Items) == 0 { - return false - } - - maxVersion := 0 - var newRS *moduledeploymentv1alpha1.ModuleReplicaSet - for i := 0; i < len(replicaSetList.Items); i++ { - version, err := getRevision(&replicaSetList.Items[i]) - if err != nil { - return false - } - if version > maxVersion { - maxVersion = version - newRS = &replicaSetList.Items[i] - } - } - - // the replicas of old replicaset must be zero - for i := 0; i < len(replicaSetList.Items); i++ { - if version, _ := getRevision(&replicaSetList.Items[i]); version != maxVersion { - if replicaSetList.Items[i].Spec.Replicas != 0 { - return false - } - } - } - - // the replicas of new replicaset must be equal to newModuleDeployment - return newRS != nil && - newRS.Spec.Template.Spec.Module.Version == "1.0.1" && - newRS.Status.Replicas == newRS.Spec.Replicas && - newRS.Status.Replicas == newModuleDeployment.Spec.Replicas + return checkModuleDeploymentReplicas( + types.NamespacedName{Name: moduleDeploymentName, Namespace: namespace}, + newModuleDeployment.Spec.Replicas) }, timeout, interval).Should(BeTrue()) }) }) @@ -193,7 +160,6 @@ var _ = Describe("ModuleDeployment Controller", func() { }) Context("test batchConfirm strategy", func() { - namespace := "default" moduleDeploymentName := "module-deployment-test-for-batch-confirm" nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} moduleDeployment := prepareModuleDeployment(namespace, moduleDeploymentName) @@ -203,7 +169,7 @@ var _ = Describe("ModuleDeployment Controller", func() { It("0. prepare 2 pods", func() { Eventually(func() bool { - pod := preparePod("fake-pod-3") + pod := preparePod(namespace, "fake-pod-3") pod.Labels[fmt.Sprintf("%s-%s", label.ModuleNameLabel, "dynamic-provider")] = "1.0.0" if err := k8sClient.Create(context.TODO(), &pod); err != nil { return false @@ -230,32 +196,10 @@ var _ = Describe("ModuleDeployment Controller", func() { return false } - set := map[string]string{ - label.ModuleDeploymentLabel: moduleDeployment.Name, - } - replicaSetList := &moduledeploymentv1alpha1.ModuleReplicaSetList{} - err := k8sClient.List(context.TODO(), replicaSetList, &client.ListOptions{LabelSelector: labels.SelectorFromSet(set)}, client.InNamespace(moduleDeployment.Namespace)) - if err != nil || len(replicaSetList.Items) == 0 { - return false - } - - maxVersion := 0 - var newRS *moduledeploymentv1alpha1.ModuleReplicaSet - for i := 0; i < len(replicaSetList.Items); i++ { - version, err := getRevision(&replicaSetList.Items[i]) - if err != nil { - return false - } - if version > maxVersion { - maxVersion = version - newRS = &replicaSetList.Items[i] - } - } - - // the replicas of new replicaset must be equal to newModuleDeployment - return newRS != nil && - newRS.Status.Replicas == newRS.Spec.Replicas && - newRS.Status.Replicas == 1 + return checkModuleDeploymentReplicas( + types.NamespacedName{ + Name: moduleDeploymentName, + Namespace: moduleDeployment.Namespace}, 1) }, timeout, interval).Should(BeTrue()) }) @@ -311,6 +255,35 @@ var _ = Describe("ModuleDeployment Controller", func() { }) }) +func checkModuleDeploymentReplicas(nn types.NamespacedName, replicas int32) bool { + set := map[string]string{ + label.ModuleDeploymentLabel: nn.Name, + } + replicaSetList := &moduledeploymentv1alpha1.ModuleReplicaSetList{} + err := k8sClient.List(context.TODO(), replicaSetList, &client.ListOptions{LabelSelector: labels.SelectorFromSet(set)}, client.InNamespace(nn.Namespace)) + if err != nil || len(replicaSetList.Items) == 0 { + return false + } + + maxVersion := 0 + var newRS *moduledeploymentv1alpha1.ModuleReplicaSet + for i := 0; i < len(replicaSetList.Items); i++ { + version, err := getRevision(&replicaSetList.Items[i]) + if err != nil { + return false + } + if version > maxVersion { + maxVersion = version + newRS = &replicaSetList.Items[i] + } + } + + // the replicas of new replicaset must be equal to newModuleDeployment + return newRS != nil && + newRS.Status.Replicas == newRS.Spec.Replicas && + newRS.Status.Replicas == replicas +} + func prepareModuleDeployment(namespace, moduleDeploymentName string) v1alpha1.ModuleDeployment { baseDeploymentName := "dynamic-stock-deployment" From 2f45cbb235ea7d731f80ff00da0194124fcaaf33 Mon Sep 17 00:00:00 2001 From: charlie Date: Thu, 28 Sep 2023 14:58:54 +0800 Subject: [PATCH 08/12] remove test=true in makefile --- module-controller/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module-controller/Makefile b/module-controller/Makefile index 0ba88d6c1..d7a66b03e 100644 --- a/module-controller/Makefile +++ b/module-controller/Makefile @@ -62,7 +62,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - test=true KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out ##@ Build From 2a13c384b61f8fa0f5288c3e8ac89d073fd02c9d Mon Sep 17 00:00:00 2001 From: charlie Date: Thu, 28 Sep 2023 16:04:41 +0800 Subject: [PATCH 09/12] test: add useBeta unit test --- .../controller/moduledeployment_controller.go | 34 ++++++++------ .../moduledeployment_controller_suit_test.go | 44 ++++++++++++++++++- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index bbd399a2f..e8e9d05de 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -110,7 +110,7 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting: - return r.updateModuleReplicaSet(moduleDeployment, newRS, oldRSs) + return r.updateModuleReplicaSet(ctx, moduleDeployment, newRS, oldRSs) case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted: if moduleDeployment.Spec.Replicas != newRS.Spec.Replicas { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting @@ -309,11 +309,9 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicas( return nil } -func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *moduledeploymentv1alpha1.ModuleDeployment, +func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, moduleDeployment *moduledeploymentv1alpha1.ModuleDeployment, newRS *moduledeploymentv1alpha1.ModuleReplicaSet, oldRSs []*moduledeploymentv1alpha1.ModuleReplicaSet) (ctrl.Result, error) { var ( - ctx = context.TODO() - batchCount = moduleDeployment.Spec.OperationStrategy.BatchCount curBatch = moduleDeployment.Status.ReleaseStatus.CurrentBatch @@ -350,7 +348,8 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo replicas := int32(0) // use beta strategy - if batchCount != 1 && curBatch == 1 && moduleDeployment.Spec.OperationStrategy.UseBeta { + useBeta := batchCount != 1 && curBatch == 1 && moduleDeployment.Spec.OperationStrategy.UseBeta + if useBeta { replicas = 1 } else if curBatch == batchCount { // if it's the last batch replicas = expReplicas @@ -363,11 +362,27 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo return ctrl.Result{}, err } + var message string now := metav1.Now() - moduleDeployment.Status.ReleaseStatus.CurrentBatch += 1 + + if useBeta { + moduleDeployment.Status.ReleaseStatus.CurrentBatch = 1 + message = "deployment release: beta deployment" + } else { + moduleDeployment.Status.ReleaseStatus.CurrentBatch += 1 + message = fmt.Sprintf("deployment release: curbatch %v, batchCount %v", curBatch, batchCount) + } + moduleDeployment.Status.ReleaseStatus.LastTransitionTime = now moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting + moduleDeployment.Status.Conditions = append(moduleDeployment.Status.Conditions, moduledeploymentv1alpha1.ModuleDeploymentCondition{ + Type: moduledeploymentv1alpha1.DeploymentProgressing, + Status: corev1.ConditionTrue, + LastTransitionTime: now, + Message: message, + }) + var grayTime int if curBatch != batchCount { if moduleDeployment.Spec.OperationStrategy.NeedConfirm { // use NeedConfirm Strategy @@ -382,13 +397,6 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(moduleDeployment *mo } } - moduleDeployment.Status.Conditions = append(moduleDeployment.Status.Conditions, moduledeploymentv1alpha1.ModuleDeploymentCondition{ - Type: moduledeploymentv1alpha1.DeploymentProgressing, - Status: corev1.ConditionTrue, - LastTransitionTime: metav1.Now(), - Message: fmt.Sprintf("deployment release: curbatch %v, batchCount %v", curBatch, batchCount), - }) - return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(grayTime) * time.Second}, r.Status().Update(ctx, moduleDeployment) } diff --git a/module-controller/internal/controller/moduledeployment_controller_suit_test.go b/module-controller/internal/controller/moduledeployment_controller_suit_test.go index a17f93d93..3aa525f0a 100644 --- a/module-controller/internal/controller/moduledeployment_controller_suit_test.go +++ b/module-controller/internal/controller/moduledeployment_controller_suit_test.go @@ -20,7 +20,7 @@ import ( ) var _ = Describe("ModuleDeployment Controller", func() { - const timeout = time.Second * 30 + const timeout = time.Second * 120 const interval = time.Second * 5 namespace := "module-deployment-namespace" @@ -225,6 +225,48 @@ var _ = Describe("ModuleDeployment Controller", func() { return moduleDeployment.Status.ReleaseStatus.Progress == moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted }, timeout, interval).Should(BeTrue()) }) + + It("5. delete moduleDeployment", func() { + Expect(k8sClient.Delete(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + }) + + Context("test useBeta strategy", func() { + moduleDeploymentName := "module-deployment-test-for-use-beta" + nn := types.NamespacedName{Namespace: namespace, Name: moduleDeploymentName} + moduleDeployment := prepareModuleDeployment(namespace, moduleDeploymentName) + moduleDeployment.Spec.Replicas = 4 + moduleDeployment.Spec.OperationStrategy.UseBeta = true + moduleDeployment.Spec.OperationStrategy.NeedConfirm = true + moduleDeployment.Spec.OperationStrategy.BatchCount = 2 + + It("0. prepare pods", func() { + Eventually(func() bool { + pod := preparePod(namespace, "fake-pod-use-beta") + pod.Labels[fmt.Sprintf("%s-%s", label.ModuleNameLabel, "dynamic-provider")] = "1.0.0" + if err := k8sClient.Create(context.TODO(), &pod); err != nil { + return false + } + // when install module, the podIP is necessary + pod.Status.PodIP = "127.0.0.1" + return k8sClient.Status().Update(context.TODO(), &pod) == nil + }, timeout, interval).Should(BeTrue()) + }) + + It("1. create a new moduleDeployment", func() { + Expect(k8sClient.Create(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + + It("2. check if use Beta strategy", func() { + Eventually(func() bool { + return checkModuleDeploymentReplicas(nn, 1) + }) + }) + + It("3. clean environment", func() { + Expect(k8sClient.Delete(context.TODO(), &moduleDeployment)).Should(Succeed()) + }) + }) Context("delete module deployment", func() { From e15a1de4dfb6b0a3885d511232ff0d2e05de5bee Mon Sep 17 00:00:00 2001 From: charlie Date: Thu, 28 Sep 2023 16:18:00 +0800 Subject: [PATCH 10/12] resolve conflict --- .../internal/controller/moduledeployment_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index e8e9d05de..7c9423ce0 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -110,7 +110,7 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting: - return r.updateModuleReplicaSet(ctx, moduleDeployment, newRS, oldRSs) + return r.updateModuleReplicaSet(ctx, moduleDeployment, newRS) case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressCompleted: if moduleDeployment.Spec.Replicas != newRS.Spec.Replicas { moduleDeployment.Status.ReleaseStatus.Progress = moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressExecuting @@ -310,7 +310,7 @@ func (r *ModuleDeploymentReconciler) updateModuleReplicas( } func (r *ModuleDeploymentReconciler) updateModuleReplicaSet(ctx context.Context, moduleDeployment *moduledeploymentv1alpha1.ModuleDeployment, - newRS *moduledeploymentv1alpha1.ModuleReplicaSet, oldRSs []*moduledeploymentv1alpha1.ModuleReplicaSet) (ctrl.Result, error) { + newRS *moduledeploymentv1alpha1.ModuleReplicaSet) (ctrl.Result, error) { var ( batchCount = moduleDeployment.Spec.OperationStrategy.BatchCount curBatch = moduleDeployment.Status.ReleaseStatus.CurrentBatch From 8b14d2695cc35f97e338b7c25289aab56b64997c Mon Sep 17 00:00:00 2001 From: charlie Date: Thu, 28 Sep 2023 16:41:13 +0800 Subject: [PATCH 11/12] fix: unit test --- .../controller/moduledeployment_controller.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index 7c9423ce0..c37567ced 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -136,12 +136,11 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } } - - if !moduleVersionChanged && isUrlChange(moduleDeployment.Spec.Template.Spec.Module, newRS.Spec.Template.Spec.Module) { - newRS.Spec.Template.Spec.Module = moduleDeployment.Spec.Template.Spec.Module - if err := r.Client.Update(ctx, newRS); err != nil { - return ctrl.Result{}, err - } + } + if !moduleVersionChanged && isUrlChange(moduleDeployment.Spec.Template.Spec.Module, newRS.Spec.Template.Spec.Module) { + newRS.Spec.Template.Spec.Module = moduleDeployment.Spec.Template.Spec.Module + if err := r.Client.Update(ctx, newRS); err != nil { + return ctrl.Result{}, err } } From c5fdd1f2a3900417bce297f1c74be58890f8e3cb Mon Sep 17 00:00:00 2001 From: charlie Date: Thu, 28 Sep 2023 17:11:14 +0800 Subject: [PATCH 12/12] fix: unit test --- .../controller/moduledeployment_controller.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/module-controller/internal/controller/moduledeployment_controller.go b/module-controller/internal/controller/moduledeployment_controller.go index c37567ced..48d9b42a4 100644 --- a/module-controller/internal/controller/moduledeployment_controller.go +++ b/module-controller/internal/controller/moduledeployment_controller.go @@ -119,6 +119,12 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } } + if !moduleVersionChanged && isUrlChange(moduleDeployment.Spec.Template.Spec.Module, newRS.Spec.Template.Spec.Module) { + newRS.Spec.Template.Spec.Module = moduleDeployment.Spec.Template.Spec.Module + if err := r.Client.Update(ctx, newRS); err != nil { + return ctrl.Result{}, err + } + } case moduledeploymentv1alpha1.ModuleDeploymentReleaseProgressWaitingForConfirmation: moduleDeployment.Spec.Pause = true if err := r.Update(ctx, moduleDeployment); err != nil { @@ -137,12 +143,6 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req } } } - if !moduleVersionChanged && isUrlChange(moduleDeployment.Spec.Template.Spec.Module, newRS.Spec.Template.Spec.Module) { - newRS.Spec.Template.Spec.Module = moduleDeployment.Spec.Template.Spec.Module - if err := r.Client.Update(ctx, newRS); err != nil { - return ctrl.Result{}, err - } - } // update moduleDeployment owner reference err = r.updateOwnerReference(ctx, moduleDeployment)