Merge pull request #134 from KevFan/parallel-tests

feat: run integration tests concurrently

Makefile

@@ -91,6 +91,8 @@ IMG ?= $(DEFAULT_IMG)
 UNIT_DIRS := ./pkg/... ./api/...
 INTEGRATION_TEST_SUITE_PATHS := ./controllers/...
 INTEGRATION_COVER_PKGS := ./pkg/...,./controllers/...,./api/...
+INTEGRATION_TEST_NUM_CORES ?= 4
+INTEGRATION_TEST_NUM_PROCESSES ?= 10
 
 # Limitador Operator replaced version
 DEFAULT_REPLACES_VERSION = 0.0.0-alpha
@@ -228,8 +230,15 @@ test-integration: clean-cov generate fmt vet ginkgo ## Run Integration tests.
 		--coverpkg $(INTEGRATION_COVER_PKGS) \
 		--output-dir $(PROJECT_PATH)/coverage/integration \
 		--coverprofile cover.out \
-		--fail-fast \
 		-v \
+		--compilers=$(INTEGRATION_TEST_NUM_CORES) \
+		--procs=$(INTEGRATION_TEST_NUM_PROCESSES) \
+		--randomize-all \
+		--randomize-suites \
+		--fail-on-pending \
+		--keep-going \
+		--race \
+		--trace \
 		$(INTEGRATION_TEST_SUITE_PATHS)
 
 ifdef TEST_NAME

controllers/limitador_controller_affinity_test.go

@@ -23,7 +23,7 @@ var _ = Describe("Limitador controller manages affinity", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -55,7 +55,7 @@ var _ = Describe("Limitador controller manages affinity", func() {
 			limitadorObj.Spec.Affinity = affinity
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create a new deployment with the custom affinity", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}
@@ -98,7 +98,7 @@ var _ = Describe("Limitador controller manages affinity", func() {
 			limitadorObj = basicLimitador(testNamespace)
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify the deployment with the affinity custom settings", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}

controllers/limitador_controller_limits_sync_test.go

@@ -31,7 +31,7 @@ var _ = Describe("Limitador controller syncs limits to pod", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -63,7 +63,7 @@ var _ = Describe("Limitador controller syncs limits to pod", func() {
 			limitadorObj.Spec.Limits = limits
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should annotate limitador pods with annotation of limits cm resource version", func(ctx SpecContext) {
 			podList := &corev1.PodList{}
@@ -117,7 +117,7 @@ var _ = Describe("Limitador controller syncs limits to pod", func() {
 			limitadorObj.Spec.Limits = limits
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should update limitador pods annotation and sync config map to pod", func(ctx SpecContext) {
 			// Check cm resource version of pods before update

controllers/limitador_controller_limits_test.go

@@ -23,7 +23,7 @@ var _ = Describe("Limitador controller manages limits", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -55,7 +55,7 @@ var _ = Describe("Limitador controller manages limits", func() {
 			limitadorObj.Spec.Limits = limits
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create configmap with the custom limits", func(ctx SpecContext) {
 			cm := &v1.ConfigMap{}
@@ -99,7 +99,7 @@ var _ = Describe("Limitador controller manages limits", func() {
 			limitadorObj = basicLimitador(testNamespace)
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify configmap with the new limits", func(ctx SpecContext) {
 			cm := &v1.ConfigMap{}

controllers/limitador_controller_pdb_test.go

@@ -24,7 +24,7 @@ var _ = Describe("Limitador controller manages PodDisruptionBudget", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -41,7 +41,7 @@ var _ = Describe("Limitador controller manages PodDisruptionBudget", func() {
 			limitadorObj.Spec.PodDisruptionBudget = pdbType
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create PodDisruptionBudget", func(ctx SpecContext) {
 			pdb := &policyv1.PodDisruptionBudget{}

controllers/limitador_controller_ports_test.go

@@ -26,7 +26,7 @@ var _ = Describe("Limitador controller manages ports", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -48,7 +48,7 @@ var _ = Describe("Limitador controller manages ports", func() {
 			}
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should configure k8s resources with the custom ports", func(ctx SpecContext) {
 			// Deployment ports
@@ -142,7 +142,7 @@ var _ = Describe("Limitador controller manages ports", func() {
 			limitadorObj = basicLimitador(testNamespace)
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify the k8s resources with the custom ports", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}

controllers/limitador_controller_replicas_test.go

@@ -22,7 +22,7 @@ var _ = Describe("Limitador controller manages replicas", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -38,7 +38,7 @@ var _ = Describe("Limitador controller manages replicas", func() {
 			limitadorObj.Spec.Replicas = ptr.To(int(replicas))
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create a new deployment with the custom replicas", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}
@@ -62,8 +62,8 @@ var _ = Describe("Limitador controller manages replicas", func() {
 		BeforeEach(func(ctx SpecContext) {
 			limitadorObj = basicLimitador(testNamespace)
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
-			Eventually(testLimitadorIsReady(ctx, limitadorObj), time.Minute, 5*time.Second).Should(Succeed())
-		}, nodeTimeOut)
+			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
+		})
 
 		It("Should modify deployment replicas", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}

controllers/limitador_controller_resources_test.go

@@ -23,7 +23,7 @@ var _ = Describe("Limitador controller manages resource requirements", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -41,7 +41,7 @@ var _ = Describe("Limitador controller manages resource requirements", func() {
 			limitadorObj.Spec.ResourceRequirements = &resourceRequirements
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create a new deployment with the custom resource requirements", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}
@@ -70,7 +70,7 @@ var _ = Describe("Limitador controller manages resource requirements", func() {
 			limitadorObj = basicLimitador(testNamespace)
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify deployment resource requirements", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}

controllers/limitador_controller_test.go

@@ -38,7 +38,7 @@ var _ = Describe("Limitador controller", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -51,7 +51,7 @@ var _ = Describe("Limitador controller", func() {
 			limitadorObj = basicLimitador(testNamespace)
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create a Limitador service with default ports", func(ctx SpecContext) {
 			createdLimitadorService := corev1.Service{}
@@ -182,7 +182,7 @@ var _ = Describe("Limitador controller", func() {
 
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create a new deployment with rate limit headers command line arg", func(ctx SpecContext) {
 			createdLimitadorDeployment := appsv1.Deployment{}
@@ -220,7 +220,7 @@ var _ = Describe("Limitador controller", func() {
 
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify the limitador deployment command line args", func(ctx SpecContext) {
 			updatedLimitador := limitadorv1alpha1.Limitador{}
@@ -271,7 +271,7 @@ var _ = Describe("Limitador controller", func() {
 
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify the limitador deployment command line args", func(ctx SpecContext) {
 			Eventually(func(g Gomega) {
@@ -325,7 +325,7 @@ var _ = Describe("Limitador controller", func() {
 
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should create a new deployment with verbosity level command line arg", func(ctx SpecContext) {
 			deployment := &appsv1.Deployment{}
@@ -371,7 +371,7 @@ var _ = Describe("Limitador controller", func() {
 
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify the limitador deployment command line args", func(ctx SpecContext) {
 			deployment := &appsv1.Deployment{}
@@ -438,7 +438,7 @@ var _ = Describe("Limitador controller", func() {
 			limitadorObj = basicLimitador(testNamespace)
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("User tries adding side-cars to deployment CR", func(ctx SpecContext) {
 			deploymentObj := appsv1.Deployment{}
@@ -507,7 +507,7 @@ var _ = Describe("Limitador controller", func() {
 			Eventually(func(g Gomega) {
 				g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(redisSecret), secret)).To(Succeed())
 			}).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("command line is correct", func(ctx SpecContext) {
 			limitadorObj := limitadorWithRedisStorage(client.ObjectKeyFromObject(redisSecret), testNamespace)
@@ -563,7 +563,7 @@ var _ = Describe("Limitador controller", func() {
 			Eventually(func(g Gomega) {
 				g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(redisSecret), secret)).To(Succeed())
 			}).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("with all defaults, the command line is correct", func(ctx SpecContext) {
 			limitadorObj := limitadorWithRedisCachedStorage(client.ObjectKeyFromObject(redisSecret), testNamespace)

controllers/limitador_controller_version_test.go

@@ -23,7 +23,7 @@ var _ = Describe("Limitador controller manages image version", func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		CreateNamespaceWithContext(ctx, &testNamespace)
-	}, nodeTimeOut)
+	})
 
 	AfterEach(func(ctx SpecContext) {
 		DeleteNamespaceWithContext(ctx, &testNamespace)
@@ -37,7 +37,7 @@ var _ = Describe("Limitador controller manages image version", func() {
 			limitadorObj.Spec.Version = ptr.To("otherversion")
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			// Do not expect to have limitador ready
-		}, nodeTimeOut)
+		})
 
 		It("Should create a new deployment with the custom image", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}
@@ -61,7 +61,7 @@ var _ = Describe("Limitador controller manages image version", func() {
 
 			Expect(k8sClient.Create(ctx, limitadorObj)).Should(Succeed())
 			Eventually(testLimitadorIsReady(ctx, limitadorObj)).WithContext(ctx).Should(Succeed())
-		}, nodeTimeOut)
+		})
 
 		It("Should modify the deployment with the custom image", func(ctx SpecContext) {
 			deployment := appsv1.Deployment{}