Merge pull request stolostron#367 from zhiweiyin318/fix-webhook

fix webhook cert rotation

Makefile

@@ -65,6 +65,12 @@ deploy-foundation-hub: ensure-kustomize
 	$(KUSTOMIZE) build deploy/foundation/hub | $(KUBECTL) apply -f -
 	mv deploy/foundation/hub/kustomization.yaml.tmp deploy/foundation/hub/kustomization.yaml
 
+deploy-foundation-webhook: ensure-kustomize
+	cp deploy/foundation/hub/resources/webhook/kustomization.yaml deploy/foundation/hub/resources/webhook/kustomization.yaml.tmp
+	cd deploy/foundation/hub/resources/webhook && ../../../../../$(KUSTOMIZE) edit set image foundation-webhook=$(FOUNDATION_IMAGE_NAME)
+	$(KUSTOMIZE) build deploy/foundation/hub/resources/webhook | $(KUBECTL) apply -f -
+	mv deploy/foundation/hub/resources/webhook/kustomization.yaml.tmp deploy/foundation/hub/resources/webhook/kustomization.yaml
+
 deploy-foundation-agent: ensure-kustomize
 	cp deploy/foundation/klusterlet/kustomization.yaml deploy/foundation/klusterlet/kustomization.yaml.tmp
 	cd deploy/foundation/klusterlet && ../../../$(KUSTOMIZE) edit set image foundation-agent=$(FOUNDATION_IMAGE_NAME)
@@ -78,10 +84,15 @@ clean-foundation-hub:
 clean-foundation-agent:
 	$(KUBECTL) delete -k deploy/foundation/klusterlet
 
+clean-foundation-webhook:
+	$(KUBECTL) delete -k deploy/foundation/hub/resources/webhook
+
+clean-deploy: clean-foundation-agent clean-foundation-webhook clean-foundation-hub
+
 build-e2e:
 	go test -c ./test/e2e
 
-test-e2e: build-e2e deploy-hub deploy-klusterlet deploy-foundation-hub deploy-foundation-agent
+test-e2e: build-e2e deploy-hub deploy-klusterlet deploy-foundation-hub deploy-foundation-webhook deploy-foundation-agent
 	./e2e.test -test.v -ginkgo.v
 
 ############################################################

cmd/webhook/app/options/options.go

@@ -4,9 +4,10 @@ package options
 
 import (
 	"crypto/tls"
+	"sync"
+	"time"
 
 	"github.com/spf13/pflag"
-	"k8s.io/klog"
 )
 
 // Config contains the server (the webhook) cert and key.
@@ -43,14 +44,53 @@ func (c *Options) AddFlags(fs *pflag.FlagSet) {
 		"Maximum burst for throttle.")
 }
 
-func ConfigTLS(o *Options) *tls.Config {
-	sCert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
-	if err != nil {
-		klog.Fatal(err)
+type certificateCacheEntry struct {
+	cert  *tls.Certificate
+	err   error
+	birth time.Time
+}
+
+// isStale returns true when this cache entry is too old to be usable
+func (c *certificateCacheEntry) isStale() bool {
+	return time.Since(c.birth) > time.Second
+}
+
+func newCertificateCacheEntry(certFile, keyFile string) certificateCacheEntry {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	return certificateCacheEntry{cert: &cert, err: err, birth: time.Now()}
+}
+
+// cachingCertificateLoader ensures that we don't hammer the filesystem when opening many connections
+// the underlying cert files are read at most once every second
+func cachingCertificateLoader(certFile, keyFile string) func() (*tls.Certificate, error) {
+	current := newCertificateCacheEntry(certFile, keyFile)
+	var currentMtx sync.RWMutex
+
+	return func() (*tls.Certificate, error) {
+		currentMtx.RLock()
+		if current.isStale() {
+			currentMtx.RUnlock()
+
+			currentMtx.Lock()
+			defer currentMtx.Unlock()
+
+			if current.isStale() {
+				current = newCertificateCacheEntry(certFile, keyFile)
+			}
+		} else {
+			defer currentMtx.RUnlock()
+		}
+
+		return current.cert, current.err
 	}
+}
 
+func ConfigTLS(o *Options) *tls.Config {
+	dynamicCertLoader := cachingCertificateLoader(o.CertFile, o.KeyFile)
 	return &tls.Config{
-		MinVersion:   tls.VersionTLS12,
-		Certificates: []tls.Certificate{sCert},
+		MinVersion: tls.VersionTLS12,
+		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			return dynamicCertLoader()
+		},
 	}
 }

deploy/foundation/hub/resources/webhook/kustomization.yaml

@@ -0,0 +1,11 @@
+resources:
+- webhook.yaml
+- webhook-service.yaml
+- webhook-validating-config.yaml
+
+images:
+- name: foundation-webhook
+  newName: quay.io/open-cluster-management/multicloud-manager
+  newTag: latest
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization

deploy/foundation/hub/resources/webhook/webhook-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    "service.beta.openshift.io/serving-cert-secret-name": foundation-webhook
+  labels:
+    app: foundation-webhook
+  name: foundation-webhook
+  namespace: open-cluster-management
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 8000
+  selector:
+    app: foundation-webhook
+  type: ClusterIP

deploy/foundation/hub/resources/webhook/webhook-validating-config.yaml

@@ -0,0 +1,45 @@
+# Copyright Contributors to the Open Cluster Management project
+
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  annotations:
+    service.beta.openshift.io/inject-cabundle: "true"
+  labels:
+    app: foundation-webhook
+  name: foundation-validating-webhook
+webhooks:
+  - name: ocm.validating.webhook.admission.open-cluster-management.io
+    admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: foundation-webhook
+        namespace: open-cluster-management
+        path: /validating
+        port: 443
+    failurePolicy: Fail
+    matchPolicy: Equivalent
+    rules:
+      - apiGroups:
+          - hive.openshift.io
+        operations:
+          - CREATE
+          - UPDATE
+        apiVersions:
+          - "v1"
+        resources:
+          - clusterdeployments
+          - clusterpools
+        scope: '*'
+      - apiGroups:
+          - cluster.open-cluster-management.io
+        operations:
+          - CREATE
+          - UPDATE
+        apiVersions:
+          - "*"
+        resources:
+          - managedclusters
+    sideEffects: None
+    timeoutSeconds: 10