webhook for clusteroperations

api/apis/clusteroperation/v1alpha1/types.go

@@ -115,7 +115,6 @@ const (
 	RunningStatus   OpsStatus = "Running"
 	SucceededStatus OpsStatus = "Succeeded"
 	FailedStatus    OpsStatus = "Failed"
-	BlockedStatus   OpsStatus = "Blocked"
 )
 
 // Status contains information about the current status of a

api/constants/constants.go

@@ -4,6 +4,8 @@ const InfoManifestGlobal = "manifest-global"
 
 const KubeanClusterLabelKey = "clusterName"
 
+const KubeanClusterHasCompleted = "hasCompleted"
+
 const Hosts_yml = "hosts.yml"
 
 const Group_vars_yml = "group_vars.yml"

charts/kubean/templates/clusterrole.yaml

@@ -20,5 +20,8 @@ rules:
     resources: ['*']
     verbs: ['*']
   - apiGroups: ['authorization.k8s.io']
+    resources: ['*']
+    verbs: ['*']
+  - apiGroups: ['admissionregistration.k8s.io']
     resources: ['*']
     verbs: ['*']

charts/kubean/templates/deployment.yaml

@@ -32,10 +32,22 @@ spec:
             {{- toYaml .Values.kubeanOperator.securityContext | nindent 12 }}
           image: "{{ .Values.kubeanOperator.image.registry }}/{{ .Values.kubeanOperator.image.repository }}:{{ .Values.kubeanOperator.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.kubeanOperator.image.pullPolicy }}
+          env:
+            - name: MUTATE_WEBHOOK_CONFIG
+              value: kubean-admission-mutate
+            - name: WEBHOOK_SERVICE_NAME
+              value: {{ include "kubean.fullname" . }}
+            - name: WEBHOOK_SERVICE_NAMESPACE
+              value: {{ include "kubean.namespace" . }}
+            - name: WEBHOOK_FAILURE_POLICY
+              value: Ignore
           ports:
             - name: http
               containerPort: 80
               protocol: TCP
+            - name: webhook-port
+              containerPort: 10443
+              protocol: TCP
           resources:
             {{- toYaml .Values.kubeanOperator.resources | nindent 12 }}
       {{- with .Values.kubeanOperator.nodeSelector }}

charts/kubean/templates/service.yaml

@@ -12,5 +12,9 @@ spec:
       targetPort: http
       protocol: TCP
       name: http
+    - port: 443
+      targetPort: webhook-port
+      protocol: TCP
+      name: webhook-port
   selector:
     {{- include "kubean.selectorLabels" . | nindent 4 }}

charts/kubean/templates/webhook.yaml

@@ -0,0 +1,4 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: kubean-admission-mutate

cmd/kubean-operator/app/app.go

@@ -9,11 +9,13 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"time"
 
 	kubeanClusterClientSet "github.com/kubean-io/kubean-api/generated/cluster/clientset/versioned"
 	kubeanClusterOperationClientSet "github.com/kubean-io/kubean-api/generated/clusteroperation/clientset/versioned"
 	kubeanLocalArtifactSetClientSet "github.com/kubean-io/kubean-api/generated/localartifactset/clientset/versioned"
 	kubeaninfomanifestClientSet "github.com/kubean-io/kubean-api/generated/manifest/clientset/versioned"
+
 	"github.com/kubean-io/kubean/pkg/controllers/cluster"
 	"github.com/kubean-io/kubean/pkg/controllers/clusterops"
 	"github.com/kubean-io/kubean/pkg/controllers/infomanifest"
@@ -87,45 +89,70 @@ func StartManager(ctx context.Context, opt *Options) error {
 		klog.Errorf("Failed to add health check endpoint: %s", err)
 		return err
 	}
-	if err := setupManager(controllerManager, opt, ctx.Done()); err != nil {
+	ClientSet, clusterClientSet, clusterClientOperationSet, infomanifestClientSet, localArtifactSetClientSet, err := prepareClient()
+	if err != nil {
+		klog.ErrorS(err, "create clientSet")
+		return err
+	}
+	if err := setupManager(controllerManager, ClientSet, clusterClientSet, clusterClientOperationSet, infomanifestClientSet, localArtifactSetClientSet, opt, ctx.Done()); err != nil {
 		klog.Errorf("setupManager %s", err)
 		return err
 	}
+	go func() {
+		for {
+			if err := clusterops.RunWithLeaseLock(ctx, ClientSet, clusterClientOperationSet); err != nil {
+				klog.ErrorS(err, "run webhook with LeaseLock")
+			}
+			time.Sleep(time.Second * 5)
+		}
+	}()
 	if err := controllerManager.Start(ctx); err != nil {
 		klog.Errorf("KubeanOperator ControllerManager exit ,%s", err)
 		return err
 	}
 	return nil
 }
 
-func setupManager(mgr controllerruntime.Manager, opt *Options, stopChan <-chan struct{}) error {
+func prepareClient() (kubernetes.Interface, kubeanClusterClientSet.Interface, kubeanClusterOperationClientSet.Interface, kubeaninfomanifestClientSet.Interface, kubeanLocalArtifactSetClientSet.Interface, error) {
 	resetConfig, err := rest.InClusterConfig()
 	if err != nil {
 		resetConfig, err = clientcmd.BuildConfigFromFlags("", os.Getenv("HOME")+"/.kube/config")
 		if err != nil {
-			return err
+			return nil, nil, nil, nil, nil, err
 		}
 	}
 	ClientSet, err := kubernetes.NewForConfig(resetConfig)
 	if err != nil {
-		return err
+		return nil, nil, nil, nil, nil, err
 	}
 	clusterClientSet, err := kubeanClusterClientSet.NewForConfig(resetConfig)
 	if err != nil {
-		return err
+		return nil, nil, nil, nil, nil, err
 	}
 	clusterClientOperationSet, err := kubeanClusterOperationClientSet.NewForConfig(resetConfig)
 	if err != nil {
-		return err
+		return nil, nil, nil, nil, nil, err
 	}
 	infomanifestClientSet, err := kubeaninfomanifestClientSet.NewForConfig(resetConfig)
 	if err != nil {
-		return err
+		return nil, nil, nil, nil, nil, err
 	}
 	localArtifactSetClientSet, err := kubeanLocalArtifactSetClientSet.NewForConfig(resetConfig)
 	if err != nil {
-		return err
+		return nil, nil, nil, nil, nil, err
 	}
+	return ClientSet, clusterClientSet, clusterClientOperationSet, infomanifestClientSet, localArtifactSetClientSet, nil
+}
+
+func setupManager(
+	mgr controllerruntime.Manager,
+	ClientSet kubernetes.Interface,
+	clusterClientSet kubeanClusterClientSet.Interface,
+	clusterClientOperationSet kubeanClusterOperationClientSet.Interface,
+	infomanifestClientSet kubeaninfomanifestClientSet.Interface,
+	localArtifactSetClientSet kubeanLocalArtifactSetClientSet.Interface,
+	opt *Options, stopChan <-chan struct{},
+) error {
 	clusterController := &cluster.Controller{
 		Client:              mgr.GetClient(),
 		ClientSet:           ClientSet,

hack/run-nightly-cluster-e2e.sh

@@ -139,21 +139,21 @@ CLUSTER_OPERATION_NAME1="cluster1-install-"`date "+%H-%M-%S"`
 sed -i "s/e2e-cluster1-install/${CLUSTER_OPERATION_NAME1}/"  "${dest_config_path}"/kubeanClusterOps.yml
 
 
-## prepare cluster upgrade job yml --> upgrade from v1.25.3 to v1.26.0
+## prepare cluster upgrade job yml --> upgrade from v1.26.3 to v1.26.4
 dest_config_path="${REPO_ROOT}"/test/kubean_sonobouy_nightlye2e/e2e-upgrade-cluster-y/
 func_prepare_config_yaml "${SOURCE_CONFIG_PATH}"  "${dest_config_path}"
 CLUSTER_OPERATION_NAME2="cluster1-upgrade-y"
 sed -i "s/e2e-cluster1-install/${CLUSTER_OPERATION_NAME2}/"  "${dest_config_path}"/kubeanClusterOps.yml
 sed -i "s/cluster.yml/upgrade-cluster.yml/" "${dest_config_path}"/kubeanClusterOps.yml
-sed -i "s/v1.25.3/v1.26.0/"  "${dest_config_path}"/vars-conf-cm.yml
+sed -i "s/v1.26.3/v1.26.4/"  "${dest_config_path}"/vars-conf-cm.yml
 
-## prepare cluster upgrade job yml --> upgrade from v1.25.3 to v1.26.7
+## prepare cluster upgrade job yml --> upgrade from v1.26.3 to v1.26.7
 dest_config_path="${REPO_ROOT}"/test/kubean_sonobouy_nightlye2e/e2e-upgrade-cluster-z/
 func_prepare_config_yaml "${SOURCE_CONFIG_PATH}"  "${dest_config_path}"
 CLUSTER_OPERATION_NAME3="cluster1-upgrade-z"
 sed -i "s/e2e-cluster1-install/${CLUSTER_OPERATION_NAME3}/"  "${dest_config_path}"/kubeanClusterOps.yml
 sed -i "s/cluster.yml/upgrade-cluster.yml/" "${dest_config_path}"/kubeanClusterOps.yml
-sed -i "s/v1.25.3/v1.26.7/"  "${dest_config_path}"/vars-conf-cm.yml
+sed -i "s/v1.26.3/v1.26.7/"  "${dest_config_path}"/vars-conf-cm.yml
 
 ginkgo -v -race -timeout=6h --fail-fast ./test/kubean_sonobouy_nightlye2e/  -- --kubeconfig="${KUBECONFIG_FILE}" \
           --clusterOperationName="${CLUSTER_OPERATION_NAME1}"  --vmipaddr="${vm_ip_addr1}" --vmipaddr2="${vm_ip_addr2}" \

hack/yml_utils.sh

@@ -76,7 +76,7 @@ function yamlUtil::abstract_groupVars() {
     yq -i '.metrics_server_enabled = "true"'  $groupVarYml
     yq -i '.local_path_provisioner_enabled = "true"'  $groupVarYml
     yq -i '.kubeadm_init_timeout = "600s"'  $groupVarYml
-    yq -i '.kube_version = "v1.25.3"'  $groupVarYml
+    yq -i '.kube_version = "v1.26.3"'  $groupVarYml
 }
 
 function yamlUtil::update_groupVars() {