Skip to content

Commit

Permalink
Setup webhook in integration test
Browse files Browse the repository at this point in the history
  • Loading branch information
SaschaSchwarze0 committed Sep 23, 2023
1 parent fb93486 commit 8620a2c
Show file tree
Hide file tree
Showing 7 changed files with 228 additions and 10 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,12 @@ jobs:
kubectl -n tekton-pipelines rollout status deployment tekton-pipelines-webhook --timeout=1m
- name: Test
run: |
# host.docker.internal does not work in a GitHub action
docker exec kind-control-plane bash -c "echo '172.17.0.1 host.docker.internal' >>/etc/hosts"
# Build and load the Git image
export GIT_CONTAINER_IMAGE="$(KO_DOCKER_REPO=kind.local ko publish ./cmd/git)"
make test-integration
e2e:
Expand Down
16 changes: 13 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ TEST_NAMESPACE ?= default
TEKTON_VERSION ?= v0.44.0

# E2E test flags
TEST_E2E_FLAGS ?= --fail-fast -p --randomize-all -timeout=1h -trace -vv
TEST_E2E_FLAGS ?= -p --randomize-all -timeout=1h -trace -v

# E2E test service account name to be used for the build runs, can be set to generated to use the generated service account feature
TEST_E2E_SERVICEACCOUNT_NAME ?= pipeline
Expand Down Expand Up @@ -204,14 +204,14 @@ test-unit-ginkgo: ginkgo
# Based on https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/integration-tests.md
.PHONY: test-integration
test-integration: install-apis ginkgo
./hack/setup-webhook-cert-integration-test.sh
$(GINKGO) \
--randomize-all \
--randomize-suites \
--fail-on-pending \
-trace \
test/integration/...


.PHONY: test-e2e
test-e2e: install-strategies test-e2e-plain

Expand All @@ -237,7 +237,17 @@ install-with-pprof:
GOOS=$(GO_OS) GOARCH=$(GO_ARCH) GOFLAGS="$(GO_FLAGS) -tags=pprof_enabled" ko apply -R -f deploy/ -- --server-side

install-apis:
kubectl apply -f deploy/crds/ --server-side
for resource in buildruns builds buildstrategies clusterbuildstrategies ; do \
if kubectl get crd "$${resource}.shipwright.io" >/dev/null 2>&1 ; then \
if [ "$$(kubectl get crd "$${resource}.shipwright.io" -o go-template='{{.spec.conversion.webhook.clientConfig.caBundle}}')" == "<no value>" ] ; then \
kubectl apply -f "deploy/crds/shipwright.io_$${resource}.yaml" --server-side ; \
else \
kubectl replace -f "deploy/crds/shipwright.io_$${resource}.yaml" ; \
fi ; \
else \
kubectl create -f "deploy/crds/shipwright.io_$${resource}.yaml" ; \
fi ; \
done
for i in 1 2 3 ; do \
kubectl wait --timeout=$(TIMEOUT) --for="condition=Established" crd/clusterbuildstrategies.shipwright.io && \
break ; \
Expand Down
81 changes: 81 additions & 0 deletions hack/setup-webhook-cert-integration-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/bin/bash

# Copyright The Shipwright Contributors
#
# SPDX-License-Identifier: Apache-2.0

set -euo pipefail

if ! hash jq >/dev/null 2>&1 ; then
echo "[ERROR] jq is not installed"
exit 1
fi

if ! hash openssl >/dev/null 2>&1 ; then
echo "[ERROR] openssl is not installed"
exit 1
fi

echo "[INFO] Generating key and signing request for Shipwright Build Webhook"

cat <<EOF >/tmp/csr.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = host.docker.internal
EOF

openssl genrsa -out /tmp/server-key.pem 2048
openssl req -new -days 365 -key /tmp/server-key.pem -subj "/O=system:nodes/CN=system:node:host.docker.internal" -out /tmp/server.csr -config /tmp/csr.conf

echo "[INFO] Deleting previous CertificateSigningRequest"
kubectl delete csr shipwright-build-webhook-csr --ignore-not-found

echo "[INFO] Create a CertificateSigningRequest"
cat <<EOF | kubectl create -f -
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
name: shipwright-build-webhook-csr
spec:
groups:
- system:authenticated
request: $(base64 </tmp/server.csr | tr -d '\n')
signerName: kubernetes.io/kubelet-serving
usages:
- digital signature
- key encipherment
- server auth
EOF

echo "[INFO] Approve the CertificateSigningRequest"
kubectl certificate approve shipwright-build-webhook-csr

certificate="$(kubectl get csr shipwright-build-webhook-csr -o json | jq -r '.status.certificate')"
while [ "${certificate}" == "null" ]; do
echo "[INFO] Waiting for certificate to be ready"
sleep 1
certificate="$(kubectl get csr shipwright-build-webhook-csr -o json | jq -r '.status.certificate')"
done

openssl base64 -d -A -out /tmp/server-cert.pem <<<"${certificate}"

echo "[INFO] Deleting the CertificateSigningRequest"
kubectl delete csr shipwright-build-webhook-csr --ignore-not-found
rm -rf /tmp/csr.conf

echo "[INFO] Retrieving CABundle"
CA="$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n')"

echo "[INFO] Patching caBundle into CustomResourceDefinitions"
kubectl patch crd clusterbuildstrategies.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]"
kubectl patch crd buildstrategies.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]"
kubectl patch crd builds.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]"
kubectl patch crd buildruns.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]"
16 changes: 14 additions & 2 deletions test/integration/integration_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package integration_test

import (
"fmt"
"net/http"
"testing"

. "github.com/onsi/ginkgo/v2"
Expand All @@ -28,10 +29,21 @@ func TestIntegration(t *testing.T) {
// TODO: clean resources in cluster, e.g. mainly cluster-scope ones
// TODO: clean each resource created per spec
var (
tb *utils.TestBuild
err error
tb *utils.TestBuild
err error
webhookServer *http.Server
)

var _ = BeforeSuite(func() {
webhookServer = utils.StartBuildWebhook()
})

var _ = AfterSuite(func() {
if webhookServer != nil {
utils.StopBuildWebhook(webhookServer)
}
})

var _ = BeforeEach(func() {
tb, err = utils.NewTestBuild()
if err != nil {
Expand Down
7 changes: 3 additions & 4 deletions test/utils/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
buildClient "github.com/shipwright-io/build/pkg/client/clientset/versioned"
"github.com/shipwright-io/build/pkg/ctxlog"
"github.com/shipwright-io/build/test"
// from https://github.com/kubernetes/client-go/issues/345
)

var (
Expand All @@ -42,11 +41,11 @@ type TestBuild struct {
Interval time.Duration
TimeOut time.Duration
KubeConfig *rest.Config
Clientset *kubernetes.Clientset
Clientset kubernetes.Interface
Namespace string
StopBuildControllers context.CancelFunc
BuildClientSet *buildClient.Clientset
PipelineClientSet *tektonClient.Clientset
BuildClientSet buildClient.Interface
PipelineClientSet tektonClient.Interface
ControllerRuntimeClient client.Client
Catalog test.Catalog
Context context.Context
Expand Down
3 changes: 2 additions & 1 deletion test/utils/namespaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package utils

import (
"context"
"time"

corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -68,5 +69,5 @@ func (t *TestBuild) DeleteNamespace() error {
return false, nil
}

return wait.PollImmediate(t.Interval, t.TimeOut, pollNamespace)
return wait.PollImmediate(t.Interval, 5*time.Second, pollNamespace)
}
110 changes: 110 additions & 0 deletions test/utils/webhook.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Copyright The Shipwright Contributors
//
// SPDX-License-Identifier: Apache-2.0

package utils

import (
"context"
"crypto/tls"
"net/http"
"time"

"github.com/shipwright-io/build/pkg/webhook/conversion"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
)

func StartBuildWebhook() *http.Server {
mux := http.NewServeMux()
mux.HandleFunc("/convert", conversion.CRDConvertHandler(context.Background()))
mux.HandleFunc("/health", health)

webhookServer := &http.Server{
Addr: ":30443",
Handler: mux,
ReadHeaderTimeout: 32 * time.Second,
IdleTimeout: time.Second,
TLSConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.X25519},
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
},
},
}

// start server
go func() {
defer ginkgo.GinkgoRecover()

if err := webhookServer.ListenAndServeTLS("/tmp/server-cert.pem", "/tmp/server-key.pem"); err != nil {
if err != http.ErrServerClosed {
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
}
}()

client := &http.Client{
Transport: &http.Transport{
IdleConnTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
// #nosec:G402 test code
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
TLSHandshakeTimeout: 5 * time.Second,
},
}

gomega.Eventually(func() int {
r, err := client.Get("https://localhost:30443/health")
if err != nil {
return 0
}
if r != nil {
return r.StatusCode
}
return 0
}).WithTimeout(10 * time.Second).Should(gomega.Equal(http.StatusNoContent))

return webhookServer
}

func StopBuildWebhook(webhookServer *http.Server) {
err := webhookServer.Close()
gomega.Expect(err).ToNot(gomega.HaveOccurred())

client := &http.Client{
Transport: &http.Transport{
IdleConnTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
// #nosec:G402 test code
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
TLSHandshakeTimeout: 5 * time.Second,
},
}

gomega.Eventually(func() int {
r, err := client.Get("https://localhost:30443/health")
if err != nil {
return 0
}
if r != nil {
return r.StatusCode
}
return 0
}).WithTimeout(10 * time.Second).Should(gomega.Equal(0))
}

func health(resp http.ResponseWriter, _ *http.Request) {
resp.WriteHeader(http.StatusNoContent)
}

0 comments on commit 8620a2c

Please sign in to comment.