Tests for Tolerations on Build and BuildRun objects

Signed-off-by: Dylan Orzel <[email protected]>

.github/workflows/ci.yml

@@ -181,7 +181,7 @@ jobs:
           version: v0.24.0
           node_image: kindest/node:${{ matrix.kubernetes }}
           cluster_name: kind
-          config: test/kind/config.yaml
+          config: test/kind/config_three_node.yaml
           wait: 120s
       - name: Verify kind cluster
         run: |
@@ -190,18 +190,21 @@ jobs:
           echo "# KinD nodes:"
           kubectl get nodes
 
-          NODE_STATUS=$(kubectl get node kind-control-plane -o json | jq -r .'status.conditions[] | select(.type == "Ready") | .status')
-          if [ "${NODE_STATUS}" != "True" ]; then
-            echo "# Node is not ready:"
-            kubectl describe node kind-control-plane
+          for nodename in $(kubectl get nodes -o name); do
+            kubectl wait --for=condition=Ready=true ${nodename} --timeout=60s
+            NODE_STATUS=$(kubectl get ${nodename} -o json | jq -r .'status.conditions[] | select(.type == "Ready") | .status')
+            if [ "${NODE_STATUS}" != "True" ]; then
+              echo "# Node is not ready:"
+              kubectl describe ${nodename}
 
-            echo "# Pods:"
-            kubectl get pod -A
-            echo "# Events:"
-            kubectl get events -A
+              echo "# Pods:"
+              kubectl get pod -A
+              echo "# Events:"
+              kubectl get events -A
 
-            exit 1
-          fi
+              exit 1
+            fi
+          done
       - name: Install Tekton
         env:
           TEKTON_VERSION: ${{ matrix.tekton }}

pkg/reconciler/build/build_test.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/utils/ptr"
 	crc "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -621,7 +622,22 @@ var _ = Describe("Reconcile Build", func() {
 				buildSample.Spec.NodeSelector = map[string]string{strings.Repeat("s", 64): "amd64"}
 				buildSample.Spec.Output.PushSecret = nil
 
-				statusCall := ctl.StubFunc(corev1.ConditionFalse, build.NodeSelectorNotValid, "name part must be no more than 63 characters")
+				statusCall := ctl.StubFunc(corev1.ConditionFalse, build.NodeSelectorNotValid, "name part "+validation.MaxLenError(63))
+				statusWriter.UpdateCalls(statusCall)
+
+				_, err := reconciler.Reconcile(context.TODO(), request)
+				Expect(err).To(BeNil())
+				Expect(statusWriter.UpdateCallCount()).To(Equal(1))
+			})
+		})
+
+		Context("when Tolerations is specified", func() {
+			It("should fail to validate when the Toleration is invalid", func() {
+				// set Toleration to be invalid
+				buildSample.Spec.Tolerations = []corev1.Toleration{{Key: strings.Repeat("s", 64), Operator: "Equal", Value: "test-value"}}
+				buildSample.Spec.Output.PushSecret = nil
+
+				statusCall := ctl.StubFunc(corev1.ConditionFalse, build.TolerationNotValid, "name part "+validation.MaxLenError(63))
 				statusWriter.UpdateCalls(statusCall)
 
 				_, err := reconciler.Reconcile(context.TODO(), request)

pkg/reconciler/buildrun/buildrun_test.go

@@ -20,6 +20,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/utils/ptr"
 	knativeapi "knative.dev/pkg/apis"
@@ -1635,9 +1636,23 @@ var _ = Describe("Reconcile BuildRun", func() {
 		Context("when nodeSelector is specified", func() {
 			It("fails when the nodeSelector is invalid", func() {
 				// set nodeSelector to be invalid
-				buildSample.Spec.NodeSelector = map[string]string{strings.Repeat("s", 64): "amd64"}
+				buildRunSample.Spec.NodeSelector = map[string]string{strings.Repeat("s", 64): "amd64"}
 
-				statusCall := ctl.StubFunc(corev1.ConditionFalse, build.NodeSelectorNotValid, "must be no more than 63 characters")
+				statusCall := ctl.StubFunc(corev1.ConditionFalse, build.NodeSelectorNotValid, "name part "+validation.MaxLenError(63))
+				statusWriter.UpdateCalls(statusCall)
+
+				_, err := reconciler.Reconcile(context.TODO(), buildRunRequest)
+				Expect(err).To(BeNil())
+				Expect(statusWriter.UpdateCallCount()).To(Equal(1))
+			})
+		})
+
+		Context("when Tolerations is specified", func() {
+			It("should fail to validate when the Toleration is invalid", func() {
+				// set Toleration to be invalid
+				buildRunSample.Spec.Tolerations = []corev1.Toleration{{Key: strings.Repeat("s", 64), Operator: "Equal", Value: "test-value"}}
+
+				statusCall := ctl.StubFunc(corev1.ConditionFalse, build.TolerationNotValid, validation.MaxLenError(63))
 				statusWriter.UpdateCalls(statusCall)
 
 				_, err := reconciler.Reconcile(context.TODO(), buildRunRequest)

pkg/reconciler/buildrun/resources/taskrun_test.go

@@ -635,7 +635,7 @@ var _ = Describe("GenerateTaskrun", func() {
 
 		Context("when the build and buildrun both specify a nodeSelector", func() {
 			BeforeEach(func() {
-				build, err = ctl.LoadBuildYAML([]byte(test.MinimalBuildRunWithNodeSelector))
+				build, err = ctl.LoadBuildYAML([]byte(test.MinimalBuildWithNodeSelector))
 				Expect(err).To(BeNil())
 
 				buildRun, err = ctl.LoadBuildRunFromBytes([]byte(test.MinimalBuildRunWithNodeSelector))
@@ -654,5 +654,29 @@ var _ = Describe("GenerateTaskrun", func() {
 				Expect(got.Spec.PodTemplate.NodeSelector).To(Equal(buildRun.Spec.NodeSelector))
 			})
 		})
+
+		Context("when the build and buildrun both specify a Toleration", func() {
+			BeforeEach(func() {
+				build, err = ctl.LoadBuildYAML([]byte(test.MinimalBuildWithToleration))
+				Expect(err).To(BeNil())
+
+				buildRun, err = ctl.LoadBuildRunFromBytes([]byte(test.MinimalBuildRunWithToleration))
+				Expect(err).To(BeNil())
+
+				buildStrategy, err = ctl.LoadBuildStrategyFromBytes([]byte(test.ClusterBuildStrategyNoOp))
+				Expect(err).To(BeNil())
+			})
+
+			JustBeforeEach(func() {
+				got, err = resources.GenerateTaskRun(config.NewDefaultConfig(), build, buildRun, serviceAccountName, buildStrategy)
+				Expect(err).To(BeNil())
+			})
+
+			It("should give precedence to the Toleration values specified in the buildRun", func() {
+				Expect(got.Spec.PodTemplate.Tolerations[0].Key).To(Equal(buildRun.Spec.Tolerations[0].Key))
+				Expect(got.Spec.PodTemplate.Tolerations[0].Operator).To(Equal(buildRun.Spec.Tolerations[0].Operator))
+				Expect(got.Spec.PodTemplate.Tolerations[0].Value).To(Equal(buildRun.Spec.Tolerations[0].Value))
+			})
+		})
 	})
 })

test/data/v1beta1/build_buildah_tolerations_cr.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: shipwright.io/v1beta1
+kind: Build
+metadata:
+  name: buildah-tolerations-build
+spec:
+  source:
+    type: Git
+    git:
+      url: https://github.com/shipwright-io/sample-go
+    contextDir: docker-build
+  strategy:
+    name: buildah-shipwright-managed-push
+    kind: ClusterBuildStrategy
+  output:
+    image: image-registry.openshift-image-registry.svc:5000/build-examples/taxi-app
+  tolerations:
+    - key: "test-key"
+      value: "test-value"
+      operator: "Equal"

test/data/v1beta1/buildrun_buildah_tolerations_cr.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: shipwright.io/v1beta1
+kind: BuildRun
+metadata:
+  name: buildah-tolerations-buildrun
+spec:
+  build:
+    name: buildah-tolerations-build

test/e2e/v1beta1/e2e_test.go

@@ -15,6 +15,8 @@ import (
 
 	buildv1beta1 "github.com/shipwright-io/build/pkg/apis/build/v1beta1"
 	shpgit "github.com/shipwright-io/build/pkg/git"
+
+	corev1 "k8s.io/api/core/v1"
 )
 
 var _ = Describe("For a Kubernetes cluster with Tekton and build installed", func() {
@@ -706,4 +708,54 @@ var _ = Describe("For a Kubernetes cluster with Tekton and build installed", fun
 		})
 	})
 
+	Context("when tolerations are specified", func() {
+
+		BeforeEach(func() {
+			testID = generateTestID("tolerations")
+
+			// create the build definition
+			build = createBuild(
+				testBuild,
+				testID,
+				"test/data/v1beta1/build_buildah_tolerations_cr.yaml",
+			)
+
+			// Add a taint to both of the kind worker nodes
+			nodes, err := testBuild.GetNodes()
+			Expect(err).ToNot(HaveOccurred())
+			for _, node := range nodes.Items {
+				if node.Name == "kind-worker" || node.Name == "kind-worker2" {
+					taint := corev1.Taint{Key: "test-key", Value: "test-vaue", Effect: "NoSchedule"}
+					testBuild.AddNodeTaint(node.Name, &taint)
+				}
+			}
+		})
+
+		AfterEach(func() {
+			err := testBuild.RemoveNodeTaints("kind-worker")
+			Expect(err).ToNot(HaveOccurred())
+			err = testBuild.RemoveNodeTaints("kind-worker2")
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("successfully runs a build when it tolerates the node taint", func() {
+			buildRun, err = buildRunTestData(testBuild.Namespace, testID, "test/data/v1beta1/buildrun_buildah_tolerations_cr.yaml")
+			Expect(err).ToNot(HaveOccurred(), "Error retrieving buildrun test data")
+
+			buildRun = validateBuildRunToSucceed(testBuild, buildRun)
+			testBuild.ValidateImageDigest(buildRun)
+		})
+
+		It("fails to schedule when the build does not tolerate the node taint", func() {
+			buildRun, err = buildRunTestData(testBuild.Namespace, testID, "test/data/v1beta1/buildrun_buildah_tolerations_cr.yaml")
+			Expect(err).ToNot(HaveOccurred(), "Error retrieving buildrun test data")
+			buildRun.Spec.Tolerations[0].Key = "untolerated"
+
+			validateBuildRunToFail(testBuild, buildRun)
+			buildRun, err = testBuild.LookupBuildRun(types.NamespacedName{Name: buildRun.Name, Namespace: testBuild.Namespace})
+
+			Expect(buildRun.Status.FailureDetails.Message).To(Equal(shpgit.AuthPrompted.ToMessage()))
+			Expect(buildRun.Status.FailureDetails.Reason).To(Equal(shpgit.AuthPrompted.String()))
+		})
+	})
 })

test/integration/build_to_taskruns_test.go

@@ -246,4 +246,50 @@ var _ = Describe("Integration tests Build and TaskRun", func() {
 			})
 		})
 	})
+
+	Context("when a build with Tolerations is defined", func() {
+		BeforeEach(func() {
+			buildSample = []byte(test.MinimalBuildWithToleration)
+			buildRunSample = []byte(test.MinimalBuildRun)
+		})
+
+		Context("when the TaskRun is created", func() {
+			It("should have the Tolerations specified in the PodTemplate", func() {
+				Expect(tb.CreateBuild(buildObject)).To(BeNil())
+
+				buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+				Expect(err).To(BeNil())
+				Expect(*buildObject.Status.Message).To(Equal(v1beta1.AllValidationsSucceeded))
+				Expect(*buildObject.Status.Registered).To(Equal(corev1.ConditionTrue))
+				Expect(*buildObject.Status.Reason).To(Equal(v1beta1.SucceedStatus))
+
+				Expect(tb.CreateBR(buildRunObject)).To(BeNil())
+
+				_, err = tb.GetBRTillStartTime(buildRunObject.Name)
+				Expect(err).To(BeNil())
+
+				tr, err := tb.GetTaskRunFromBuildRun(buildRunObject.Name)
+				Expect(err).To(BeNil())
+				Expect(buildObject.Spec.Tolerations[0].Key).To(Equal(tr.Spec.PodTemplate.Tolerations[0].Key))
+				Expect(buildObject.Spec.Tolerations[0].Operator).To(Equal(tr.Spec.PodTemplate.Tolerations[0].Operator))
+				Expect(buildObject.Spec.Tolerations[0].Value).To(Equal(tr.Spec.PodTemplate.Tolerations[0].Value))
+				Expect(tr.Spec.PodTemplate.Tolerations[0].Effect).To(Equal(corev1.TaintEffectNoSchedule))
+				Expect(tr.Spec.PodTemplate.Tolerations[0].TolerationSeconds).To(Equal(corev1.Toleration{}.TolerationSeconds))
+			})
+		})
+
+		Context("when the Toleration is invalid", func() {
+			It("fails the build with a proper error in Reason", func() {
+				// set Toleration Key to be invalid
+				buildObject.Spec.Tolerations[0].Key = strings.Repeat("s", 64)
+				Expect(tb.CreateBuild(buildObject)).To(BeNil())
+
+				buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+				Expect(err).To(BeNil())
+
+				Expect(*buildObject.Status.Registered).To(Equal(corev1.ConditionFalse))
+				Expect(*buildObject.Status.Reason).To(Equal(v1beta1.TolerationNotValid))
+			})
+		})
+	})
 })

test/integration/buildruns_to_taskruns_test.go

@@ -21,6 +21,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 )
@@ -582,4 +583,54 @@ var _ = Describe("Integration tests BuildRuns and TaskRuns", func() {
 			})
 		})
 	})
+
+	Context("when a buildrun is created with a Toleration defined", func() {
+		BeforeEach(func() {
+			buildSample = []byte(test.MinimalBuild)
+			buildRunSample = []byte(test.MinimalBuildRunWithToleration)
+		})
+
+		Context("when the taskrun is created", func() {
+			It("should have the Toleration specified in the PodTemplate", func() {
+				Expect(tb.CreateBuild(buildObject)).To(BeNil())
+
+				buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+				Expect(err).To(BeNil())
+
+				Expect(tb.CreateBR(buildRunObject)).To(BeNil())
+
+				br, err := tb.GetBRTillCompletion(buildRunObject.Name)
+				Expect(err).To(BeNil())
+
+				tr, err := tb.GetTaskRunFromBuildRun(buildRunObject.Name)
+				Expect(err).To(BeNil())
+				Expect(br.Spec.Tolerations[0].Key).To(Equal(tr.Spec.PodTemplate.Tolerations[0].Key))
+				Expect(br.Spec.Tolerations[0].Operator).To(Equal(tr.Spec.PodTemplate.Tolerations[0].Operator))
+				Expect(br.Spec.Tolerations[0].Value).To(Equal(tr.Spec.PodTemplate.Tolerations[0].Value))
+				Expect(tr.Spec.PodTemplate.Tolerations[0].Effect).To(Equal(corev1.TaintEffectNoSchedule))
+				Expect(tr.Spec.PodTemplate.Tolerations[0].TolerationSeconds).To(Equal(corev1.Toleration{}.TolerationSeconds))
+			})
+		})
+
+		Context("when the Toleration is invalid", func() {
+			It("fails the buildrun with a proper error in Reason", func() {
+				Expect(tb.CreateBuild(buildObject)).To(BeNil())
+
+				buildObject, err = tb.GetBuildTillValidation(buildObject.Name)
+				Expect(err).To(BeNil())
+
+				// set Toleration Key to be invalid
+				buildRunObject.Spec.Tolerations[0].Key = strings.Repeat("s", 64)
+				Expect(tb.CreateBR(buildRunObject)).To(BeNil())
+
+				br, err := tb.GetBRTillCompletion(buildRunObject.Name)
+				Expect(err).To(BeNil())
+
+				condition := br.Status.GetCondition(v1beta1.Succeeded)
+				Expect(condition.Status).To(Equal(corev1.ConditionFalse))
+				Expect(condition.Reason).To(Equal("PodCreationFailed"))
+				Expect(condition.Message).To(ContainSubstring(validation.MaxLenError(63)))
+			})
+		})
+	})
 })

test/kind/config_three_node.yaml

@@ -0,0 +1,17 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+kubeadmConfigPatches:
+- |
+  kind: ClusterConfiguration
+  metadata:
+    name: config
+  apiServer:
+    extraArgs:
+      enable-admission-plugins: PodSecurity
+nodes:
+- role: control-plane
+  extraPortMappings:
+    - containerPort: 32222
+      hostPort: 32222
+- role: worker
+- role: worker