feat: failing deployments E2E tested

Makefile

@@ -47,12 +47,12 @@ e2e-deploy:
 .PHONY: test-e2e
 test-e2e:
 	@echo "Running e2e tests..."
-	@go test -v -race -count 1 ./test/
+	@go test -race -count 1 ./test/
 
 .PHONY: test-unit
 test-unit:
 	@echo "Running unit tests..."
-	@go test -v -race -count 1 ./webhook/
+	@go test -race -count 1 ./webhook/
 
 test-cleanup:
 	@echo "Cleaning up..."

test/framework/client.go

@@ -36,7 +36,7 @@ func New() (*Framework, error) {
 func (f *Framework) CreateDeployment(t testing.TB, d appsv1.Deployment) {
 	_, err := f.k8s.AppsV1().Deployments("test-cases").Create(context.Background(), &d, metav1.CreateOptions{})
 	if err != nil {
-		f.Cleanup(t)
+		f.Cleanup(t, err)
 	}
 }
 
@@ -50,8 +50,7 @@ func (f *Framework) WaitForDeployment(t *testing.T, ns, name string) {
 	})
 
 	if err != nil {
-		t.Fatalf("failed watching deployment: %v", err)
-		f.Cleanup(t)
+		f.Cleanup(t, err)
 	}
 	for event := range w.ResultChan() {
 		deployment, ok := event.Object.(*appsv1.Deployment)
@@ -70,10 +69,13 @@ func (f *Framework) WaitForDeployment(t *testing.T, ns, name string) {
 
 // Cleanup removes all resources created by the framework
 // and cleans up the testing directory
-func (f *Framework) Cleanup(t testing.TB) {
+func (f *Framework) Cleanup(t testing.TB, err error) {
 	cleanupKeys(t)
 	f.cleanupDeployments(t)
 	f.cleanupSecrets(t)
+	if err != nil {
+		t.Fatalf("test failed: %v", err)
+	}
 }
 
 // cleanupKeys removes all keypair files from the testing directory
@@ -97,7 +99,7 @@ func cleanupKeys(t testing.TB) {
 			}
 		}
 	}
-	t.Logf("cleaned up keypair files for")
+	t.Logf("cleaned up keypair files")
 }
 
 // CreateKeys creates a signing keypair for cosing with the provided name
@@ -111,20 +113,17 @@ func (f *Framework) CreateKeys(t testing.TB, name string) (string, string) {
 	cmd.SetArgs(args)
 	err = cmd.Execute()
 	if err != nil {
-		f.Cleanup(t)
-		t.Fatalf("failed creating keypair: %v", err)
+		f.Cleanup(t, err)
 	}
 
 	// read private key and public key from the current directory
 	privateKey, err := os.ReadFile(fmt.Sprintf("%s.key", name))
 	if err != nil {
-		f.Cleanup(t)
-		t.Fatalf("failed reading private key: %v", err)
+		f.Cleanup(t, err)
 	}
 	pubKey, err := os.ReadFile(fmt.Sprintf("%s.pub", name))
 	if err != nil {
-		f.Cleanup(t)
-		t.Fatalf("failed reading public key: %v", err)
+		f.Cleanup(t, err)
 	}
 
 	return string(privateKey), string(pubKey)
@@ -157,35 +156,35 @@ func (f *Framework) SignContainer(t *testing.T, priv, img string) {
 	_ = cmd.Flags().Set("allow-http-registry", "true")
 	err := cmd.Execute()
 	if err != nil {
-		t.Fatalf("failed signing container: %v", err)
+		f.Cleanup(t, err)
 	}
 }
 
-// cleanupDeployments removes all deployments from the testing namespace,
+// cleanupDeployments removes all deployments from the testing namespace
 // if they exist
 func (f *Framework) cleanupDeployments(t testing.TB) {
 
 	t.Logf("cleaning up deployments")
 	deployments, err := f.k8s.AppsV1().Deployments("test-cases").List(context.Background(), metav1.ListOptions{})
 	if err != nil {
-		t.Fatalf("failed listing deployments: %v", err)
+		f.Cleanup(t, err)
 	}
 	for _, d := range deployments.Items {
 		err = f.k8s.AppsV1().Deployments("test-cases").Delete(context.Background(), d.Name, metav1.DeleteOptions{})
 		if err != nil {
-			t.Fatalf("failed deleting deployment %s: %v", d.Name, err)
+			f.Cleanup(t, err)
 		}
 	}
 
 	timeout := time.After(30 * time.Second)
 	for {
 		select {
 		case <-timeout:
-			t.Fatalf("timeout reached while waiting for pods to be deleted")
+			f.Cleanup(t, fmt.Errorf("timeout reached while waiting for deployments to be deleted"))
 		default:
 			pods, err := f.k8s.CoreV1().Pods("test-cases").List(context.Background(), metav1.ListOptions{})
 			if err != nil {
-				t.Fatalf("failed listing pods: %v", err)
+				f.Cleanup(t, err)
 			}
 
 			if len(pods.Items) == 0 {
@@ -202,25 +201,92 @@ func (f *Framework) CreateSecret(t *testing.T, secret corev1.Secret) {
 	t.Logf("creating secret %s", secret.Name)
 	s, err := f.k8s.CoreV1().Secrets("test-cases").Create(context.Background(), &secret, metav1.CreateOptions{})
 	if err != nil {
-		t.Fatalf("failed creating secret: %v", err)
+		f.Cleanup(t, err)
 	}
 	t.Logf("created secret %s", s.Name)
 }
 
-// cleanupSecrets removes all secrets from the testing namespace,
+// cleanupSecrets removes all secrets from the testing namespace
 func (f *Framework) cleanupSecrets(t testing.TB) {
 
 	t.Logf("cleaning up secrets")
 	secrets, err := f.k8s.CoreV1().Secrets("test-cases").List(context.Background(), metav1.ListOptions{})
 	if err != nil {
-		t.Fatalf("failed listing secrets: %v", err)
+		f.Cleanup(t, err)
+	}
+	if len(secrets.Items) == 0 {
+		return
 	}
 	for _, s := range secrets.Items {
 		err = f.k8s.CoreV1().Secrets("test-cases").Delete(context.Background(), s.Name, metav1.DeleteOptions{})
 		if err != nil {
-			t.Fatalf("failed deleting secret %s: %v", s.Name, err)
+			f.Cleanup(t, err)
+		}
+	}
+}
+
+// AssertDeploymentFailed asserts that the deployment cannot start
+func (f *Framework) AssertDeploymentFailed(t *testing.T, d appsv1.Deployment) {
+
+	t.Logf("waiting for deployment %s to fail", d.Name)
+
+	// watch for replicasets of the deployment
+	rsName, err := f.waitForReplicaSetCreation(t, d)
+	if err != nil {
+		f.Cleanup(t, err)
+	}
+
+	// get warning events of deployment's namespace and check if the deployment failed
+	w, err := f.k8s.CoreV1().Events("test-cases").Watch(context.Background(), metav1.ListOptions{
+		FieldSelector: fmt.Sprintf("involvedObject.name=%s", rsName),
+	})
+	if err != nil {
+		f.Cleanup(t, err)
+	}
+
+	timeout := time.After(30 * time.Second)
+	for event := range w.ResultChan() {
+		select {
+		case <-timeout:
+			f.Cleanup(t, fmt.Errorf("timeout reached while waiting for deployment to fail"))
+		default:
+			e, ok := event.Object.(*corev1.Event)
+			if !ok {
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			if e.Reason == "FailedCreate" {
+				t.Logf("deployment %s failed: %s", d.Name, e.Message)
+				return
+			}
+			time.Sleep(5 * time.Second)
+		}
+	}
+}
+
+func (f *Framework) waitForReplicaSetCreation(t *testing.T, d appsv1.Deployment) (string, error) {
+	rs, err := f.k8s.AppsV1().ReplicaSets("test-cases").Watch(context.Background(), metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("app=%s", d.Name),
+	})
+	if err != nil {
+		f.Cleanup(t, err)
+	}
+
+	timeout := time.After(30 * time.Second)
+	for event := range rs.ResultChan() {
+		select {
+		case <-timeout:
+			return "", fmt.Errorf("timeout reached while waiting for replicaset to be created")
+		default:
+			rs, ok := event.Object.(*appsv1.ReplicaSet)
+			if ok {
+				t.Logf("replicaset %s created", rs.Name)
+				return rs.Name, nil
+			}
+			time.Sleep(5 * time.Second)
 		}
 	}
+	return "", fmt.Errorf("failed to wait for replicaset creation")
 }
 
 func createClientSet() (k8sClient *kubernetes.Clientset, err error) {

test/main_test.go

@@ -1,15 +1,10 @@
 package test
 
 import (
-	"os"
 	"testing"
 )
 
-func TestDeployments(t *testing.T) {
-
-	if os.Getenv("SKIP_TEST_DEPLOYMENTS") != "" {
-		t.Skip("Skipping TestDeployments")
-	}
+func TestPassingDeployments(t *testing.T) {
 
 	testFuncs := map[string]func(t *testing.T){
 		"OneContainerSinglePubKeyEnvRef":            testOneContainerSinglePubKeyEnvRef,
@@ -24,3 +19,16 @@ func TestDeployments(t *testing.T) {
 		t.Run(name, tf)
 	}
 }
+
+func TestFailingDeployments(t *testing.T) {
+
+	testFuncs := map[string]func(t *testing.T){
+		"OneContainerSinglePubKeyMalformedEnvRef":  testOneContainerSinglePubKeyMalformedEnvRef,
+		"TwoContainersSinglePubKeyMalformedEnvRef": testTwoContainersSinglePubKeyMalformedEnvRef,
+		"OneContainerSinglePubKeyNoMatchEnvRef":    testOneContainerSinglePubKeyNoMatchEnvRef,
+	}
+
+	for name, tf := range testFuncs {
+		t.Run(name, tf)
+	}
+}