feature: multi-container support (#22)

* check all container in loop * fix error messages * refactor: moved common logic to functions, fixed no pubkey logic * chore: added function documentation * refactor: validate container moved to function * chore: removed .test folder * chore: simplification of dockerfile * feat: makefile & testcases * refactor: verification of container image only * feat: added support for init containers * fix: fixed logging on server shutdown * chore: added test-cases and first draft of e2e tests * chore: added makefile to support local e2e tests * feat: first e2e test draft with signed container & deployment * feat: first working deployment test * feat: end2end.yaml should work now * fix: ephemeral key generation will work now * fix: typo in e2e tests * chore: install cosing for e2e * fix: changed key location * chore: build action runs on main branch only * feat: happy path e2e tests * feat: failing deployments E2E tested * chore: docs & verbose tests to find failure * refactor: WaitForDeployment works with deployment * fix: replaced latest with first to be more explicit * chore: removed old yaml test cases * chore: simplification of dockerfile * feat: makefile & testcases * refactor: verification of container image only * feat: added support for init containers * fix: fixed logging on server shutdown * chore: added test-cases and first draft of e2e tests * chore: added makefile to support local e2e tests * feat: first e2e test draft with signed container & deployment * feat: first working deployment test * feat: end2end.yaml should work now * fix: ephemeral key generation will work now * fix: typo in e2e tests * chore: install cosing for e2e * fix: changed key location * chore: build action runs on main branch only * feat: happy path e2e tests * feat: failing deployments E2E tested * chore: docs & verbose tests to find failure * refactor: WaitForDeployment works with deployment * fix: replaced latest with first to be more explicit * chore: removed old yaml test cases --------- Co-authored-by: Frank Kloeker <[email protected]>
eumel8 · Oct 4, 2023 · 4d79b9b · 4d79b9b
1 parent 2d4f0cd
commit 4d79b9b
Show file tree

Hide file tree

Showing 20 changed files with 1,976 additions and 622 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -1,7 +1,9 @@
-name: Docker Image CI
+name: Build Image & Chart
 
 on:
   push:
+    branches:
+      - main
 
 jobs:
   build:
@@ -12,7 +14,7 @@ jobs:
       - name: Install Cosign
         uses: sigstore/cosign-installer@main
         with:
-          cosign-release: 'v2.0.0'
+          cosign-release: 'v2.2.0'
       - name: Login Build Sign Push
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ${GHR} -u ${{ github.actor }} --password-stdin

diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml
@@ -7,11 +7,13 @@ jobs:
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v3
-    - name: Set up K3S
-      uses: debianmaster/actions-k3s@master
-      id: k3s
-      with:
-        version: 'v1.24.16-k3s1'
+    - name: Install k3d
+      run: |
+        curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+        k3d version
+    - name: Create Cluster & Registry
+      run: |
+          make e2e-cluster
     - name: Check Cluster Nodes
       run: |
         kubectl get nodes
@@ -39,53 +41,24 @@ jobs:
         else
           echo "Deployment metrics-server OK"
         fi
-    - name: Check Cluster Pods
-      run: |
-        kubectl get pods -A
     - name: Setup Helm
       run: |
         curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
         helm version
-    - name: Install Cosignwebhook
-      run: |
-        helm -n cosignwebhook upgrade -i cosignwebhook \
-          --atomic \
-          --create-namespace \
-          --timeout 300s \
-          --set logLevel=debug chart
-    - name: Check Pods
+    - name: Install Cosign
+      uses: sigstore/cosign-installer@main
+      with:
+        cosign-release: 'v2.2.0'
+    - name: Create ephemeral keys
       run: |
-        kubectl -n cosignwebhook get pods
-    - name: Check Cosignwebhook Deployment
+        make e2e-keys
+    - name: Build test image
       run: |
-        kubectl -n cosignwebhook rollout status deployment/cosignwebhook --timeout=60s
-        STATUS=$(kubectl -n cosignwebhook get deployment cosignwebhook -o jsonpath={.status.readyReplicas})
-        if [[ $STATUS  -ne  1 ]]
-        then
-          echo "Deployment cosignwebhook not ready"
-          kubectl -n cosignwebhook get events
-          exit 1
-        else
-          echo "Deployment cosignwebhook OK"
-        fi
-    - name: Deploy Demoapp
+        make e2e-images
+    - name: Install Cosignwebhook
       run: |
-        kubectl create namespace demoapp
-        kubectl -n demoapp apply -f manifests/demoapp.yaml
-    - name: Check Demoapp Deployment
+        make e2e-deploy
+    - name: Run End2End Tests
       run: |
-        kubectl -n demoapp rollout status deployment/demoapp --timeout=60s
-        STATUS=$(kubectl -n demoapp get deployment demoapp -o jsonpath={.status.readyReplicas})
-        if [[ $STATUS  -ne  1 ]]
-        then
-          echo "Deployment demoapp not ready"
-          kubectl -n demoapp get events
-          echo "Get cosignwebhook logs"
-          kubectl -n cosignwebhook logs -lapp.kubernetes.io/name=cosignwebhook
-          exit 1
-        else
-          echo "Deployment demoapp OK"
-          kubectl -n demoapp get events
-          echo "Get cosignwebhook logs"
-          kubectl -n cosignwebhook logs -lapp.kubernetes.io/name=cosignwebhook
-        fi
+        go mod download
+        make test-e2e
diff --git a/.github/workflows/gotest.yaml b/.github/workflows/gotest.yaml
@@ -5,21 +5,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
           go-version: 1.21
-
       - name: Mod Tidy
         run: go mod tidy
-
-      - name: Mod Vendor
-        run: go mod vendor
-
-      - name: Build
-        run: go build -v ./...
-
       - name: Test
-        run: go test -v ./...
+        run: make test-unit
 
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,7 @@
 cosignwebhook
 grumpywebhook
 chart/caas-values.yaml
-vendor/
+
+# the keypair used for test-signing of the webhook
+*.key
+*.pub
diff --git a/Dockerfile b/Dockerfile
@@ -2,10 +2,9 @@
 FROM golang:1.21 AS build-env
 WORKDIR /app
 COPY  . /app
-RUN useradd -u 10001 webhook
-RUN go mod tidy
-RUN go mod vendor
-RUN CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o cosignwebhook
+RUN useradd -u 10001 webhook && \
+    go mod tidy && \
+    CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o cosignwebhook
 
 FROM alpine:latest
 COPY --from=build-env /app/cosignwebhook /cosignwebhook

diff --git a/Makefile b/Makefile
@@ -0,0 +1,70 @@
+#############
+### TESTS ###
+#############
+.PHONY: test-e2e
+test-e2e:
+	@echo "Running e2e tests..."
+	@go test -v -race -count 1 ./test/
+
+test-cleanup:
+	@echo "Cleaning up..."
+	@helm uninstall cosignwebhook -n cosignwebhook
+	@k3d registry delete k3d-registry.localhost
+	@k3d cluster delete cosign-tests
+
+.PHONY: test-unit
+test-unit:
+	@echo "Running unit tests..."
+	@go test -v -race -count 1 ./webhook/
+
+###########
+### E2E ###
+###########
+
+e2e-cluster:
+	@echo "Creating registry..."
+	@k3d registry create registry.localhost --port 5000
+	@echo "Adding registry to cluster..."
+	@k3d cluster create cosign-tests --registry-use k3d-registry.localhost:5000
+	@echo "Create test namespace..."
+	@kubectl create namespace test-cases
+
+e2e-keys:
+	@echo "Generating cosign keys..."
+	@export COSIGN_PASSWORD="" && \
+	 cosign generate-key-pair && \
+	 cosign generate-key-pair --output-key-prefix second
+
+e2e-images:
+	@echo "Checking for cosign.key..."
+	@test -f cosign.key || (echo "cosign.key not found. Run 'make generate-key' to generate one." && exit 1)
+	@echo "Building test image..."
+	@docker build -t k3d-registry.localhost:5000/cosignwebhook:dev .
+	@echo "Pushing test image..."
+	@docker push k3d-registry.localhost:5000/cosignwebhook:dev
+	@echo "Signing test image..."
+	@export COSIGN_PASSWORD="" && \
+		cosign sign --tlog-upload=false --key cosign.key k3d-registry.localhost:5000/cosignwebhook:dev
+	@echo "Importing test image to cluster..."
+	@k3d image import k3d-registry.localhost:5000/cosignwebhook:dev --cluster cosign-tests
+	@echo "Building busybox image..."
+	@docker pull busybox:latest
+	@echo "Tagging & pushing busybox images..."
+	@docker tag busybox:latest k3d-registry.localhost:5000/busybox:first
+	@docker tag busybox:latest k3d-registry.localhost:5000/busybox:second
+	@docker push k3d-registry.localhost:5000/busybox --all-tags
+	@echo "Signing busybox images..."
+	@export COSIGN_PASSWORD="" && \
+		cosign sign --tlog-upload=false --key cosign.key k3d-registry.localhost:5000/busybox:first && \
+		cosign sign --tlog-upload=false --key second.key k3d-registry.localhost:5000/busybox:second
+
+e2e-deploy:
+	@echo "Deploying test image..."
+	@helm upgrade -i cosignwebhook chart -n cosignwebhook --create-namespace \
+		--set image.repository=k3d-registry.localhost:5000/cosignwebhook \
+		--set image.tag=dev \
+		--set-file cosign.scwebhook.key=cosign.pub \
+		--set logLevel=debug \
+		--wait --debug
+
+e2e-prep: e2e-cluster e2e-keys e2e-images e2e-deploy
diff --git a/README.md b/README.md
@@ -34,6 +34,8 @@ kubectl -n cosignwebhook apply -f manifests/manifest.yaml
 
 ## Cert generation
 
+Run the generate-certs script in the `hack` folder to generate the TLS key pair and the CA certificate for the webhook:
+
 ```bash
 generate-certs.sh --service cosignwebhook --webhook cosignwebhook --namespace cosignwebhook --secret cosignwebhook
 ```
@@ -101,17 +103,29 @@ public key used to sign the image you're deploying.
 
 # Test
 
-Based on the signed image and the corresponding key, the demo app should appear or denied (check event log)
+To test the webhook, you may run the following command(s):
 
 ```bash
-kubectl create namespace cosignwebhook
-kubectl -n cosignwebhook apply -f manifests/demoapp.yaml
+# unit tests
+make test-unit
+
+# E2E tests
+make e2e-prep
+make test-e2e
 ```
 
+## E2E tests
+
+The E2E tests require a running kubernetes cluster. Currently, the namespace and webhook are deployed via helper make targets. To run the tests the following is required:
+
+- docker
+- cosign (v2)
+
 # TODO
 
 * [x] Support private images
 * [x] Support multiple container/keys
+* [ ] Support COSING_REPOSITORY
 
 # Local build
 
@@ -122,6 +136,7 @@ CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o cosignw
 ## Credits
 
 Frank Kloeker [email protected]
+Bruno Bressi, [email protected]
 
 Life is for sharing. If you have an issue with the code or want to improve it, feel free to open an issue or an pull
 request.

diff --git a/cosign.pub b/cosign.pub