Merge branch 'master' into update_to_k8s126

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Change Log
 
+## v1.5.2
+> Chang log since v1.5.1
+
+### CVE FIX: Enhance kruise-daemon security ([#1482](https://github.com/openkruise/kruise/pull/1482), [veophi](https://github.com/veophi))
+
+### Start kruise-manager as a non-root user
+We start kruise-manger with a non-root user to further enhance the security of kruise-manager. ([#1491](https://github.com/openkruise/kruise/pull/1491), [@zmberg](https://github.com/zmberg))
+
 ## v1.5.1
 > Chang log since v1.5.0
 
@@ -73,6 +81,11 @@ And some bugs were fixed by
 ([#1348](https://github.com/openkruise/kruise/pull/1348), [#1343](https://github.com/openkruise/kruise/pull/1343), [Colvin-Y](https://github.com/Colvin-Y)),
 thanks!
 
+## v.1.4.1
+> Change log since v1.4.0
+
+### CVE FIX: Enhance kruise-daemon security ([#1482](https://github.com/openkruise/kruise/pull/1482), [veophi](https://github.com/veophi))
+
 ## v1.4.0
 
 > Change log since v1.3.0
@@ -123,6 +136,12 @@ For more detail, please refer to its [proposal](https://github.com/openkruise/kr
 - Change kruise base image to alpine. ([#1166](https://github.com/openkruise/kruise/pull/1166), [@fengshunli](https://github.com/fengshunli))
 - PersistentPodState support custom workload (like statefulSet). ([#1063](https://github.com/openkruise/kruise/pull/1063), [@baxiaoshi](https://github.com/baxiaoshi))
 
+## v1.3.1
+
+> Change log since v1.3.0
+
+### CVE FIX: Enhance kruise-daemon security ([#1482](https://github.com/openkruise/kruise/pull/1482), [veophi](https://github.com/veophi))
+
 ## v1.3.0
 
 > Change log since v1.2.0

Dockerfile

@@ -23,10 +23,28 @@ ARG BASE_IMAGE
 ARG BASE_IMAGE_VERSION
 FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
 
-RUN apk add --no-cache ca-certificates bash expat \
-  && rm -rf /var/cache/apk/*
-
 WORKDIR /
 COPY --from=builder /workspace/manager .
 COPY --from=builder /workspace/daemon ./kruise-daemon
+
+RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    chown -R nobody:nobody /manager && \
+    apk --no-cache --update upgrade && \
+    apk --no-cache add ca-certificates && \
+    apk --no-cache add tzdata && \
+    rm -rf /var/cache/apk/* && \
+    update-ca-certificates && \
+    echo "only include root and nobody user" && \
+    echo -e "root:x:0:0:root:/root:/bin/ash\nnobody:x:65534:65534:nobody:/:/sbin/nologin" | tee /etc/passwd && \
+    echo -e "root:x:0:root\nnobody:x:65534:" | tee /etc/group && \
+    rm -rf /usr/local/sbin/* && \
+    rm -rf /usr/local/bin/* && \
+    rm -rf /usr/sbin/* && \
+    rm -rf /usr/bin/* && \
+    rm -rf /sbin/* && \
+    rm -rf /bin/*
+
 ENTRYPOINT ["/manager"]

Dockerfile_multiarch

@@ -26,7 +26,15 @@ ARG BASE_IMAGE
 ARG BASE_IMAGE_VERSION
 FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
 
+WORKDIR /
+COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/daemon ./kruise-daemon
+
 RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    chown -R nobody:nobody /manager && \
     apk --no-cache --update upgrade && \
     apk --no-cache add ca-certificates && \
     apk --no-cache add tzdata && \
@@ -42,7 +50,4 @@ RUN set -eux; \
     rm -rf /sbin/* && \
     rm -rf /bin/*
 
-WORKDIR /
-COPY --from=builder /workspace/manager .
-COPY --from=builder /workspace/daemon ./kruise-daemon
 ENTRYPOINT ["/manager"]

apis/apps/pub/launch_priority.go

@@ -29,4 +29,8 @@ const (
 	ContainerLaunchPriorityKey = "apps.kruise.io/container-launch-priority"
 	// ContainerLaunchOrdered is the annotation value that indicates containers in pod should be launched by ordinal.
 	ContainerLaunchOrdered = "Ordered"
+
+	// ContainerLaunchPriorityCompletedKey is the annotation indicates the pod has all its priorities
+	// patched into its barrier configmap.
+	ContainerLaunchPriorityCompletedKey = "apps.kruise.io/container-launch-priority-completed"
 )

apis/apps/v1alpha1/node_pod_probe_types.go

@@ -32,6 +32,8 @@ type PodProbe struct {
 	Namespace string `json:"namespace"`
 	// pod uid
 	UID string `json:"uid"`
+	// pod ip
+	IP string `json:"IP"`
 	// Custom container probe, supports Exec, Tcp, and returns the result to Pod yaml
 	Probes []ContainerProbe `json:"probes,omitempty"`
 }

config/crd/bases/apps.kruise.io_nodepodprobes.yaml

@@ -38,6 +38,9 @@ spec:
               podProbes:
                 items:
                   properties:
+                    IP:
+                      description: pod ip
+                      type: string
                     name:
                       description: pod name
                       type: string
@@ -221,6 +224,7 @@ spec:
                       description: pod uid
                       type: string
                   required:
+                  - IP
                   - name
                   - namespace
                   - uid

config/manager/manager.yaml

@@ -38,6 +38,14 @@ spec:
         - --feature-gates=AllAlpha=true
         image: controller:latest
         imagePullPolicy: Always
+        securityContext:
+          capabilities:
+            drop:
+              - all
+            add: [ 'NET_BIND_SERVICE' ]
+          allowPrivilegeEscalation: false
+          runAsNonRoot: true
+          runAsUser: 65534
         name: manager
         env:
           - name: KUBE_CACHE_MUTATION_DETECTOR
@@ -97,6 +105,12 @@ spec:
         - --feature-gates=AllAlpha=true
         image: controller:latest
         imagePullPolicy: Always
+        securityContext:
+          capabilities:
+            drop:
+              - all
+            add: [ 'NET_BIND_SERVICE' ]
+          allowPrivilegeEscalation: false
         name: daemon
         env:
         - name: KUBE_CACHE_MUTATION_DETECTOR

pkg/control/pubcontrol/pub_control_utils.go

@@ -65,9 +65,9 @@ func PodUnavailableBudgetValidatePod(pod *corev1.Pod, operation policyv1alpha1.P
 	if pod.Annotations[policyv1alpha1.PodPubNoProtectionAnnotation] == "true" {
 		klog.V(3).Infof("pod(%s/%s) contains annotations[%s]=true, then don't need check pub", pod.Namespace, pod.Name, policyv1alpha1.PodPubNoProtectionAnnotation)
 		return true, "", nil
-		// If the pod is not ready, it doesn't count towards healthy and we should not decrement
-	} else if !PubControl.IsPodReady(pod) {
-		klog.V(3).Infof("pod(%s/%s) is not ready, then don't need check pub", pod.Namespace, pod.Name)
+		// If the pod is not ready or state is inconsistent, it doesn't count towards healthy and we should not decrement
+	} else if !PubControl.IsPodReady(pod) || !PubControl.IsPodStateConsistent(pod) {
+		klog.V(3).Infof("pod(%s/%s) is not ready or state is inconsistent, then don't need check pub", pod.Namespace, pod.Name)
 		return true, "", nil
 	}
 

pkg/control/pubcontrol/pub_control_utils_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/openkruise/kruise/apis/apps/pub"
 	appspub "github.com/openkruise/kruise/apis/apps/pub"
 	policyv1alpha1 "github.com/openkruise/kruise/apis/policy/v1alpha1"
 	"github.com/openkruise/kruise/pkg/util/controllerfinder"
@@ -274,6 +275,34 @@ func TestPodUnavailableBudgetValidatePod(t *testing.T) {
 				return pubStatus
 			},
 		},
+		{
+			name: "valid delete pod, pod state is inconsistent(inplace update not completed yet), ignore",
+			getPod: func() *corev1.Pod {
+				pod := podDemo.DeepCopy()
+				pod.Annotations[pub.InPlaceUpdateStateKey] = `{"nextContainerImages":{"main":"nginx:v2"}}`
+				return pod
+			},
+			getPub: func() *policyv1alpha1.PodUnavailableBudget {
+				pub := pubDemo.DeepCopy()
+				return pub
+			},
+			operation:   policyv1alpha1.PubDeleteOperation,
+			expectAllow: true,
+		},
+		{
+			name: "valid delete pod, pod declared no protect , ignore",
+			getPod: func() *corev1.Pod {
+				pod := podDemo.DeepCopy()
+				pod.Annotations[policyv1alpha1.PodPubNoProtectionAnnotation] = "true"
+				return pod
+			},
+			getPub: func() *policyv1alpha1.PodUnavailableBudget {
+				pub := pubDemo.DeepCopy()
+				return pub
+			},
+			operation:   policyv1alpha1.PubDeleteOperation,
+			expectAllow: true,
+		},
 	}
 
 	for _, cs := range cases {

pkg/controller/cloneset/sync/cloneset_scale.go

@@ -110,16 +110,15 @@ func (r *realControl) Scale(
 	if podsToDelete := util.DiffPods(podsSpecifiedToDelete, podsInPreDelete); len(podsToDelete) > 0 {
 		newPodsToDelete, oldPodsToDelete := clonesetutils.GroupUpdateAndNotUpdatePods(podsToDelete, updateRevision)
 		klog.V(3).Infof("CloneSet %s try to delete pods specified. Delete ready limit: %d. New Pods: %v, old Pods: %v.",
-			controllerKey, diffRes.deleteAvailableLimit, util.GetPodNames(newPodsToDelete).List(), util.GetPodNames(oldPodsToDelete).List())
+			controllerKey, diffRes.deleteReadyLimit, util.GetPodNames(newPodsToDelete).List(), util.GetPodNames(oldPodsToDelete).List())
 
 		podsCanDelete := make([]*v1.Pod, 0, len(podsToDelete))
 		for _, pod := range podsToDelete {
-			// Determine pod available, since deleteAvailableLimit is also based on the pod available calculation
-			if !IsPodAvailable(coreControl, pod, updateCS.Spec.MinReadySeconds) {
+			if !isPodReady(coreControl, pod) {
 				podsCanDelete = append(podsCanDelete, pod)
-			} else if diffRes.deleteAvailableLimit > 0 {
+			} else if diffRes.deleteReadyLimit > 0 {
 				podsCanDelete = append(podsCanDelete, pod)
-				diffRes.deleteAvailableLimit--
+				diffRes.deleteReadyLimit--
 			}
 		}
 
@@ -137,17 +136,16 @@ func (r *realControl) Scale(
 		}
 
 		klog.V(3).Infof("CloneSet %s begin to scale in %d pods including %d (current rev), delete ready limit: %d",
-			controllerKey, diffRes.scaleDownNum, diffRes.scaleDownNumOldRevision, diffRes.deleteAvailableLimit)
+			controllerKey, diffRes.scaleDownNum, diffRes.scaleDownNumOldRevision, diffRes.deleteReadyLimit)
 
 		podsPreparingToDelete := r.choosePodsToDelete(updateCS, diffRes.scaleDownNum, diffRes.scaleDownNumOldRevision, notUpdatedPods, updatedPods)
 		podsToDelete := make([]*v1.Pod, 0, len(podsPreparingToDelete))
 		for _, pod := range podsPreparingToDelete {
-			// Determine pod available, since deleteAvailableLimit is also based on the pod available calculation
-			if !IsPodAvailable(coreControl, pod, updateCS.Spec.MinReadySeconds) {
+			if !isPodReady(coreControl, pod) {
 				podsToDelete = append(podsToDelete, pod)
-			} else if diffRes.deleteAvailableLimit > 0 {
+			} else if diffRes.deleteReadyLimit > 0 {
 				podsToDelete = append(podsToDelete, pod)
-				diffRes.deleteAvailableLimit--
+				diffRes.deleteReadyLimit--
 			}
 		}
 

pkg/controller/cloneset/sync/cloneset_scale_test.go

@@ -609,8 +609,8 @@ func TestScale(t *testing.T) {
 				}
 				return generatePods(obj, 5)
 			},
-			expectedPodsLen:  3,
-			expectedModified: true,
+			expectedPodsLen:  5,
+			expectedModified: false,
 		},
 		{
 			name: "cloneSet(replicas=3,maxUnavailable=20%,partition=nil,maxSurge=nil,minReadySeconds=0), specified delete pod-0, pods=5, and scale replicas 5 -> 3",

pkg/controller/cloneset/sync/cloneset_sync_utils.go

@@ -63,9 +63,9 @@ type expectationDiffs struct {
 	// scaleUpLimit is the limit number of creating Pods when scaling up
 	// it is limited by scaleStrategy.maxUnavailable
 	scaleUpLimit int
-	// deleteAvailableLimit is the limit number of ready Pods that can be deleted
+	// deleteReadyLimit is the limit number of ready Pods that can be deleted
 	// it is limited by UpdateStrategy.maxUnavailable
-	deleteAvailableLimit int
+	deleteReadyLimit int
 
 	// useSurge is the number that temporarily expect to be above the desired replicas
 	useSurge int
@@ -253,7 +253,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu
 		res.scaleDownNumOldRevision = integer.IntMax(currentTotalOldCount-toDeleteOldRevisionCount-expectedTotalOldCount, 0)
 	}
 	if toDeleteNewRevisionCount > 0 || toDeleteOldRevisionCount > 0 || res.scaleDownNum > 0 {
-		res.deleteAvailableLimit = integer.IntMax(maxUnavailable+(len(pods)-replicas)-totalUnavailable, 0)
+		res.deleteReadyLimit = integer.IntMax(maxUnavailable+(len(pods)-replicas)-totalUnavailable, 0)
 	}
 
 	// The consistency between scale and update will be guaranteed by syncCloneSet and expectations
@@ -281,6 +281,10 @@ func isSpecifiedDelete(cs *appsv1alpha1.CloneSet, pod *v1.Pod) bool {
 	return false
 }
 
+func isPodReady(coreControl clonesetcore.Control, pod *v1.Pod) bool {
+	return IsPodAvailable(coreControl, pod, 0)
+}
+
 func IsPodAvailable(coreControl clonesetcore.Control, pod *v1.Pod, minReadySeconds int32) bool {
 	state := lifecycle.GetPodLifecycleState(pod)
 	if state != "" && state != appspub.LifecycleStateNormal {