diff --git a/.gitignore b/.gitignore
index e4bc958..8202194 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,7 @@
 cosignwebhook
 grumpywebhook
 chart/caas-values.yaml
-test/keys
+
+# the keypair used for test-signing of the webhook
+*.key
+*.pub
diff --git a/Makefile b/Makefile
index 88ad2ef..c510d4d 100644
--- a/Makefile
+++ b/Makefile
@@ -28,14 +28,16 @@ test-busybox-images:
 
 test-image:
 	@echo "Checking for cosign.key..."
-	@test -f test/keys/cosign.key || (echo "cosign.key not found. Run 'make generate-key' to generate one." && exit 1)
+	@test -f cosign.key || (echo "cosign.key not found. Run 'make generate-key' to generate one." && exit 1)
 	@echo "Building test image..."
 	@docker build -t k3d-registry.localhost:5000/cosignwebhook:dev .
 	@echo "Pushing test image..."
 	@docker push k3d-registry.localhost:5000/cosignwebhook:dev
 	@echo "Signing test image..."
-	@export COSIGN_PASSWORD="" && \
-		cosign sign --tlog-upload=false --key cosign.key k3d-registry.localhost:5000/cosignwebhook:dev
+	@SHA=$(shell docker inspect --format='{{index .RepoDigests 0}}' k3d-registry.localhost:5000/cosignwebhook:dev | cut -d '@' -f 2) && \
+		echo "Using image SHA: $${SHA}" && \
+		export COSIGN_PASSWORD="" && \
+		cosign sign --tlog-upload=false --key cosign.key k3d-registry.localhost:5000/cosignwebhook:dev@$${SHA}
 
 test-deploy:
 	@echo "Deploying test image..."
diff --git a/README.md b/README.md
index 66d0f42..0e3cef1 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,8 @@ kubectl -n cosignwebhook apply -f manifests/manifest.yaml
 
 ## Cert generation
 
+Run the generate-certs script in the `hack` folder to generate the TLS key pair and the CA certificate for the webhook:
+
 ```bash
 generate-certs.sh --service cosignwebhook --webhook cosignwebhook --namespace cosignwebhook --secret cosignwebhook
 ```
diff --git a/generate-certs.sh b/hack/generate-certs.sh
similarity index 100%
rename from generate-certs.sh
rename to hack/generate-certs.sh
diff --git a/test/e2e_test.go b/test/e2e_test.go
index 38e081b..4a27d68 100644
--- a/test/e2e_test.go
+++ b/test/e2e_test.go
@@ -26,6 +26,7 @@ func createKeys(t testing.TB, name string) (string, string) {
 	cmd.SetArgs(args)
 	err = cmd.Execute()
 	if err != nil {
+		cleanupKeys(t, name)
 		t.Fatalf("failed generating keypair: %v", err)
 	}
 
@@ -42,14 +43,67 @@ func createKeys(t testing.TB, name string) (string, string) {
 	return string(privateKey), string(pubKey)
 }
 
-// TestOneContainerPubKeyEnvVar tests that deployment with a single signed container,
+// cleanupKeys removes all keypair files with the passed name from the testing directory
+func cleanupKeys(t testing.TB, name string) {
+
+	t.Logf("cleaning up keypair files for %s", name)
+	// check if the keypair files exist
+	_, err := os.Stat(fmt.Sprintf("%s.key", name))
+	if err != nil {
+		t.Fatalf("failed reading private key: %v", err)
+	}
+
+	_, err = os.Stat(fmt.Sprintf("%s.pub", name))
+	if err != nil {
+		t.Fatalf("failed reading public key: %v", err)
+	}
+
+	err = os.Remove(fmt.Sprintf("%s.key", name))
+	if err != nil {
+		t.Fatalf("failed removing private key: %v", err)
+	}
+	err = os.Remove(fmt.Sprintf("%s.pub", name))
+	if err != nil {
+		t.Fatalf("failed removing public key: %v", err)
+	}
+	t.Logf("cleaned up keypair files for %s", name)
+}
+
+// signContainer signs the container with the provided private key
+// TODO: find a way to simplify this function - maybe use cosing CLI directly?
+func signContainer(t *testing.T, priv, img string) error {
+	args := []string{
+		"sign",
+		img,
+	}
+	t.Setenv("COSIGN_PASSWORD", "")
+	cmd := cli.New()
+	_ = cmd.Flags().Set("timeout", "30s")
+	cmd.SetArgs(args)
+
+	// find the sign subcommand in the commands slice
+	for _, c := range cmd.Commands() {
+		if c.Name() == "sign" {
+			cmd = c
+			break
+		}
+	}
+	_ = cmd.Flags().Set("key", fmt.Sprintf("%s.key", priv))
+	_ = cmd.Flags().Set("tlog-upload", "false")
+	_ = cmd.Flags().Set("yes", "true")
+	_ = cmd.Flags().Set("allow-http-registry", "true")
+	return cmd.Execute()
+}
+
+// TestOneContainerPubKeyEnvVar tests that a deployment with a single signed container,
 // with a public key provided via an environment variable, succeeds.
 func TestOneContainerPubKeyEnvVar(t *testing.T) {
 	// create a keypair to sign the container
 	_, pub := createKeys(t, "test")
-	os.Setenv("COSIGN_PASSWORD", "")
-	// sign the container
-	err := signContainer(t, "test")
+	t.Setenv("COSIGN_PASSWORD", "")
+
+	// sign the container with the ephemeral keypair
+	err := signContainer(t, "test", "k3d-registry.localhost:5000/busybox:dev@sha256:023917ec6a886d0e8e15f28fb543515a5fcd8d938edb091e8147db4efed388ee")
 	if err != nil {
 		t.Fatalf("failed signing container: %v", err)
 	}
@@ -72,7 +126,12 @@ func TestOneContainerPubKeyEnvVar(t *testing.T) {
 					Containers: []corev1.Container{
 						{
 							Name:  "test-case-1",
-							Image: "image_name:tag",
+							Image: "k3d-registry.localhost:5000/busybox:dev@sha256:023917ec6a886d0e8e15f28fb543515a5fcd8d938edb091e8147db4efed388ee",
+							Command: []string{
+								"sh",
+								"-c",
+								"while true; do echo 'hello world, i am tired and will sleep now'; sleep 10; done",
+							},
 							Env: []corev1.EnvVar{
 								{
 									Name:  webhook.CosignEnvVar,
@@ -89,25 +148,39 @@ func TestOneContainerPubKeyEnvVar(t *testing.T) {
 	// create clientset
 	k8sClient, err := createClientSet()
 	if err != nil {
+		cleanupKeys(t, "test")
 		t.Fatalf("failed creating clientset: %v", err)
 	}
 
 	// create the deployment
 	_, err = k8sClient.AppsV1().Deployments("test-cases").Create(context.Background(), &depl, metav1.CreateOptions{})
 	if err != nil {
+		cleanupKeys(t, "test")
 		t.Fatalf("failed creating deployment: %v", err)
 	}
 
 	// wait for the deployment to be ready
 	err = waitForDeploymentReady(t, k8sClient, "test-cases", "test-case-1")
 	if err != nil {
+		cleanupKeys(t, "test")
 		t.Fatalf("failed waiting for deployment to be ready: %v", err)
 	}
+
+	// delete the deployment
+	err = k8sClient.AppsV1().Deployments("test-cases").Delete(context.Background(), "test-case-1", metav1.DeleteOptions{})
+	if err != nil {
+		cleanupKeys(t, "test")
+		t.Fatalf("failed deleting deployment: %v", err)
+	}
+
+	// cleanup the keypair
+	cleanupKeys(t, "test")
 }
 
 // waitForDeploymentReady waits for the deployment to be ready
 func waitForDeploymentReady(t *testing.T, k8sClient *kubernetes.Clientset, ns, name string) error {
 
+	t.Logf("waiting for deployment %s to be ready", name)
 	// wait until the deployment is ready
 	w, err := k8sClient.AppsV1().Deployments(ns).Watch(context.Background(), metav1.ListOptions{
 		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
@@ -116,7 +189,6 @@ func waitForDeploymentReady(t *testing.T, k8sClient *kubernetes.Clientset, ns, n
 	if err != nil {
 		return err
 	}
-
 	for event := range w.ResultChan() {
 		deployment, ok := event.Object.(*appsv1.Deployment)
 		if !ok {
@@ -124,6 +196,7 @@ func waitForDeploymentReady(t *testing.T, k8sClient *kubernetes.Clientset, ns, n
 		}
 
 		if deployment.Status.ReadyReplicas == 1 {
+			t.Logf("deployment %s is ready", name)
 			return nil
 		}
 	}
@@ -131,18 +204,6 @@ func waitForDeploymentReady(t *testing.T, k8sClient *kubernetes.Clientset, ns, n
 	return nil
 }
 
-func signContainer(t *testing.T, priv string) error {
-	args := []string{
-		"sign",
-		"--key", fmt.Sprintf("%s.key", priv),
-		"image_name:tag",
-	}
-	cmd := cli.Sign()
-	cmd.SetArgs(args)
-	return cmd.Execute()
-
-}
-
 func createClientSet() (k8sClient *kubernetes.Clientset, err error) {
 	kubeconfig := os.Getenv("KUBECONFIG")
 	if kubeconfig == "" {