diff --git a/Makefile b/Makefile
index fb48b7e8c..fc6a9d11a 100644
--- a/Makefile
+++ b/Makefile
@@ -93,6 +93,7 @@ build-e2e:
 	go test -c ./test/e2e
 
 test-e2e: build-e2e deploy-hub deploy-klusterlet deploy-foundation-hub deploy-foundation-webhook deploy-foundation-agent
+	deploy/foundation/scripts/install-check.sh
 	./e2e.test -test.v -ginkgo.v
 
 ############################################################
diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go
index 424a8122f..e400c1100 100644
--- a/cmd/agent/agent.go
+++ b/cmd/agent/agent.go
@@ -4,6 +4,7 @@ package main
 
 import (
 	"context"
+	"k8s.io/component-base/logs"
 	"os"
 	"time"
 
@@ -68,6 +69,8 @@ func main() {
 	o := options.NewAgentOptions()
 	o.AddFlags(pflag.CommandLine)
 
+	logs.InitLogs()
+	defer logs.FlushLogs()
 	ctx := signals.SetupSignalHandler()
 	startManager(o, ctx)
 }
diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go
index f2ceb03cc..b39f71a95 100644
--- a/cmd/controller/app/options/options.go
+++ b/cmd/controller/app/options/options.go
@@ -10,6 +10,7 @@ import (
 type ControllerRunOptions struct {
 	KubeConfig           string
 	CAFile               string
+	LogCertSecret        string
 	EnableInventory      bool
 	EnableLeaderElection bool
 	EnableRBAC           bool
@@ -22,6 +23,7 @@ func NewControllerRunOptions() *ControllerRunOptions {
 	return &ControllerRunOptions{
 		KubeConfig:           "",
 		CAFile:               "/var/run/agent/ca.crt",
+		LogCertSecret:        "ocm-klusterlet-self-signed-secrets",
 		EnableInventory:      true,
 		EnableLeaderElection: true,
 		EnableRBAC:           false,
@@ -36,6 +38,8 @@ func (o *ControllerRunOptions) AddFlags(fs *pflag.FlagSet) {
 		"The kubeconfig to connect to cluster to watch/apply resources.")
 	fs.StringVar(&o.CAFile, "agent-cafile", o.CAFile, ""+
 		"Agent CA file.")
+	fs.StringVar(&o.LogCertSecret, "log-cert-secret", o.LogCertSecret,
+		"log cert secret name.")
 	fs.BoolVar(&o.EnableInventory, "enable-inventory", o.EnableInventory,
 		"enable multi-cluster inventory")
 	fs.BoolVar(&o.EnableLeaderElection, "enable-leader-election", o.EnableLeaderElection,
diff --git a/cmd/controller/app/server.go b/cmd/controller/app/server.go
index 6a8999126..d2e1e409f 100644
--- a/cmd/controller/app/server.go
+++ b/cmd/controller/app/server.go
@@ -4,8 +4,7 @@ package app
 
 import (
 	"context"
-	"io/ioutil"
-	"path"
+	"github.com/open-cluster-management/multicloud-operators-foundation/pkg/controllers/certrotation"
 	"time"
 
 	clusterv1client "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
@@ -90,11 +89,6 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 	clusterInformers := clusterv1informers.NewSharedInformerFactory(clusterClient, 10*time.Minute)
 	kubeInfomers := kubeinformers.NewSharedInformerFactory(kubeClient, 10*time.Minute)
 
-	caData, err := GetAgentCA(o.CAFile)
-	if err != nil {
-		klog.Warningf("unable to get foundation agent server CA file: %v", err)
-	}
-
 	mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
 		Scheme:                 scheme,
 		LeaderElectionID:       "foundation-controller",
@@ -139,7 +133,7 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 		}
 	}
 
-	if err = clusterinfo.SetupWithManager(mgr, caData); err != nil {
+	if err = clusterinfo.SetupWithManager(mgr, o.LogCertSecret); err != nil {
 		klog.Errorf("unable to setup clusterInfo reconciler: %v", err)
 		return err
 	}
@@ -177,6 +171,12 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 		klog.Errorf("unable to setup gc reconciler: %v", err)
 		return err
 	}
+
+	if err = certrotation.SetupWithManager(mgr, o.LogCertSecret); err != nil {
+		klog.Errorf("unable to setup cert rotation reconciler: %v", err)
+		return err
+	}
+
 	go func() {
 		<-mgr.Elected()
 		go clusterInformers.Start(ctx.Done())
@@ -196,11 +196,3 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 
 	return nil
 }
-
-func GetAgentCA(caFile string) ([]byte, error) {
-	pemBlock, err := ioutil.ReadFile(path.Clean(caFile))
-	if err != nil {
-		return nil, err
-	}
-	return pemBlock, nil
-}
diff --git a/deploy/foundation/hub/resources/clusterviewv1-apiservice.yaml b/deploy/foundation/hub/resources/clusterviewv1-apiservice.yaml
index be16881f6..deee644e4 100644
--- a/deploy/foundation/hub/resources/clusterviewv1-apiservice.yaml
+++ b/deploy/foundation/hub/resources/clusterviewv1-apiservice.yaml
@@ -2,12 +2,13 @@ apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
   name: v1.clusterview.open-cluster-management.io
+  annotations:
+    "service.beta.openshift.io/inject-cabundle": "true"
 spec:
   group: clusterview.open-cluster-management.io
   version: v1
   service:
     namespace: open-cluster-management
     name: foundation-proxyserver
-  insecureSkipTLSVerify: true
   groupPriorityMinimum: 10
   versionPriority: 20
diff --git a/deploy/foundation/hub/resources/clusterviewv1alpha1-apiservice.yaml b/deploy/foundation/hub/resources/clusterviewv1alpha1-apiservice.yaml
index 7be9e4d2d..600a11754 100644
--- a/deploy/foundation/hub/resources/clusterviewv1alpha1-apiservice.yaml
+++ b/deploy/foundation/hub/resources/clusterviewv1alpha1-apiservice.yaml
@@ -2,12 +2,13 @@ apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
   name: v1alpha1.clusterview.open-cluster-management.io
+  annotations:
+    "service.beta.openshift.io/inject-cabundle": "true"
 spec:
   group: clusterview.open-cluster-management.io
   version: v1alpha1
   service:
     namespace: open-cluster-management
     name: foundation-proxyserver
-  insecureSkipTLSVerify: true
   groupPriorityMinimum: 10
   versionPriority: 20
diff --git a/deploy/foundation/hub/resources/proxyserver-apiservice.yaml b/deploy/foundation/hub/resources/proxyserver-apiservice.yaml
index 24906637b..bb6ce8455 100644
--- a/deploy/foundation/hub/resources/proxyserver-apiservice.yaml
+++ b/deploy/foundation/hub/resources/proxyserver-apiservice.yaml
@@ -2,12 +2,13 @@ apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
   name: v1beta1.proxy.open-cluster-management.io
+  annotations:
+    "service.beta.openshift.io/inject-cabundle": "true"
 spec:
   group: proxy.open-cluster-management.io
   version: v1beta1
   service:
     namespace: open-cluster-management
     name: foundation-proxyserver
-  insecureSkipTLSVerify: true
   groupPriorityMinimum: 10000
   versionPriority: 20
diff --git a/deploy/foundation/hub/resources/proxyserver-service.yaml b/deploy/foundation/hub/resources/proxyserver-service.yaml
index fdc7d2085..ddb032885 100644
--- a/deploy/foundation/hub/resources/proxyserver-service.yaml
+++ b/deploy/foundation/hub/resources/proxyserver-service.yaml
@@ -3,6 +3,8 @@ apiVersion: v1
 metadata:
   name: foundation-proxyserver
   namespace: open-cluster-management
+  annotations:
+    "service.beta.openshift.io/serving-cert-secret-name": foundation-proxyserver
 spec:
   selector:
     app: foundation-proxyserver
diff --git a/deploy/foundation/hub/resources/proxyserver.yaml b/deploy/foundation/hub/resources/proxyserver.yaml
index 84b63d156..33571e1ad 100644
--- a/deploy/foundation/hub/resources/proxyserver.yaml
+++ b/deploy/foundation/hub/resources/proxyserver.yaml
@@ -22,9 +22,17 @@ spec:
         imagePullPolicy: IfNotPresent
         args:
           - "/proxyserver"
-          - "--agent-cert-dir=/tmp/agent-cert"
-          - "--cert-dir=/tmp/cert"
           - "--secure-port=6443"
+          - "--tls-cert-file=/var/run/apiservice/tls.crt"
+          - "--tls-private-key-file=/var/run/apiservice/tls.key"
+          - "--agent-cafile=/var/run/klusterlet/ca.crt"
+          - "--agent-certfile=/var/run/klusterlet/tls.crt"
+          - "--agent-keyfile=/var/run/klusterlet/tls.key"
+        volumeMounts:
+        - mountPath: /var/run/klusterlet
+          name: klusterlet-certs
+        - mountPath: /var/run/apiservice
+          name: apiservice-certs
         livenessProbe:
           httpGet:
             path: /healthz
@@ -38,3 +46,12 @@ spec:
             scheme: HTTPS
             port: 6443
           initialDelaySeconds: 2
+      volumes:
+      - name: klusterlet-certs
+        secret:
+          defaultMode: 420
+          secretName: ocm-klusterlet-self-signed-secrets
+      - name: apiservice-certs
+        secret:
+          defaultMode: 420
+          secretName: foundation-proxyserver
diff --git a/deploy/foundation/klusterlet/resources/agent-manifestwork.yaml b/deploy/foundation/klusterlet/resources/agent-manifestwork.yaml
index bbd5a8640..204f9f918 100644
--- a/deploy/foundation/klusterlet/resources/agent-manifestwork.yaml
+++ b/deploy/foundation/klusterlet/resources/agent-manifestwork.yaml
@@ -89,7 +89,6 @@ spec:
                 - "--agent-address=foundation-agent.open-cluster-management-agent.svc"
                 - "--agent-port=443"
                 - "--lease-duration=5"
-                - "--insecure=true"
               volumeMounts:
                 - name: hub-config
                   mountPath: /var/run/hub
diff --git a/deploy/foundation/scripts/install-check.sh b/deploy/foundation/scripts/install-check.sh
new file mode 100755
index 000000000..a262de10e
--- /dev/null
+++ b/deploy/foundation/scripts/install-check.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+
+KUBECTL=${KUBECTL:-kubectl}
+
+
+for i in {1..7}; do
+  echo "############$i  Checking foundation pods"
+  RUNNING_POD=0
+  controller=$($KUBECTL -n open-cluster-management get pods | grep foundation-controller | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+controller))
+  proxyserver=$($KUBECTL -n open-cluster-management get pods | grep foundation-proxyserver | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+proxyserver))
+  webhook=$($KUBECTL -n open-cluster-management get pods | grep foundation-webhook | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+webhook))
+  agent=$($KUBECTL -n open-cluster-management-agent get pods | grep foundation-agent | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+agent))
+
+  if [ "${RUNNING_POD}" -eq 4 ]; then
+    break
+  fi
+
+  if [ $i -eq 7 ]; then
+    echo "!!!!!!!!!!  the foundation pods are not ready within 4 minutes"
+    $KUBECTL -n open-cluster-management get pods
+    $KUBECTL -n open-cluster-management get secret
+    $KUBECTL -n open-cluster-management-agent get pods
+    $KUBECTL get mcl
+    $KUBECTL -n cluster1 get manifestworks.work.open-cluster-management.io -o yaml
+    exit 1
+  fi
+  sleep 30
+done
+
+
+echo "############  Foundation is installed successfully!!"
+
diff --git a/pkg/controllers/certrotation/certrotation_controller.go b/pkg/controllers/certrotation/certrotation_controller.go
new file mode 100644
index 000000000..f079f34d1
--- /dev/null
+++ b/pkg/controllers/certrotation/certrotation_controller.go
@@ -0,0 +1,204 @@
+package certrotation
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"fmt"
+	"github.com/open-cluster-management/multicloud-operators-foundation/pkg/utils"
+	"github.com/openshift/library-go/pkg/crypto"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/klog"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+	"time"
+)
+
+const (
+	resyncInterval = time.Minute * 10
+
+	signerName      = "ocm-klusterlet-logger"
+	defaultValidity = time.Hour * 24 * 365
+)
+
+type Reconciler struct {
+	client                 client.Client
+	scheme                 *runtime.Scheme
+	logCertSecretNamespace string
+	logCertSecretName      string
+}
+
+func SetupWithManager(mgr manager.Manager, certSecret string) error {
+	namespace, secretName, err := cache.SplitMetaNamespaceKey(certSecret)
+	if err != nil {
+		return err
+	}
+	if namespace == "" {
+		namespace, err = utils.GetComponentNamespace()
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := add(mgr, newReconciler(mgr, namespace, secretName)); err != nil {
+		klog.Errorf("Failed to create cert rotation controller, %v", err)
+		return err
+	}
+	return nil
+}
+
+// newReconciler returns a new reconcile.Reconciler
+func newReconciler(mgr manager.Manager, namespace, secretName string) reconcile.Reconciler {
+
+	return &Reconciler{
+		client:                 mgr.GetClient(),
+		scheme:                 mgr.GetScheme(),
+		logCertSecretNamespace: namespace,
+		logCertSecretName:      secretName,
+	}
+}
+
+// add adds a new Controller to mgr with r as the reconcile.Reconciler
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	// Create a new controller
+	c, err := controller.New("certrotation-controller", mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return err
+	}
+
+	err = c.Watch(&source.Kind{Type: &corev1.Secret{}}, &handler.EnqueueRequestForObject{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	certSecret := &corev1.Secret{}
+
+	err := r.client.Get(ctx, types.NamespacedName{Name: r.logCertSecretName, Namespace: r.logCertSecretNamespace}, certSecret)
+	switch {
+	case errors.IsNotFound(err):
+		certSecret.Name = r.logCertSecretName
+		certSecret.Namespace = r.logCertSecretNamespace
+		certSecret.Type = corev1.SecretTypeTLS
+		certSecret.Data = map[string][]byte{}
+		if err := newCertSecret(certSecret, signerName, defaultValidity); err != nil {
+			return ctrl.Result{}, err
+		}
+		return ctrl.Result{RequeueAfter: resyncInterval}, r.client.Create(ctx, certSecret)
+
+	case err != nil:
+		return ctrl.Result{}, err
+	}
+
+	if validateCert(certSecret) == nil {
+		return ctrl.Result{RequeueAfter: resyncInterval}, nil
+	}
+
+	klog.Infof("require a new cert secret.reason: %v", err)
+	if err := newCertSecret(certSecret, signerName, defaultValidity); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	return ctrl.Result{RequeueAfter: resyncInterval}, r.client.Update(ctx, certSecret)
+}
+
+func validateCert(secret *corev1.Secret) error {
+	if secret == nil {
+		return fmt.Errorf("invalid secret")
+	}
+	caBundle := secret.Data["ca.crt"]
+	certData := secret.Data["tls.crt"]
+
+	caCert, err := decodeCert(caBundle)
+	if err != nil {
+		return fmt.Errorf("failed to decode ca cert %v", err)
+	}
+
+	clientCert, err := decodeCert(certData)
+	if err != nil {
+		return fmt.Errorf("failed to decode client cert %v", err)
+	}
+
+	if clientCert.Issuer.CommonName == caCert.Subject.CommonName {
+		return nil
+	}
+
+	return fmt.Errorf("issuer %q not in ca bundle", clientCert.Issuer.CommonName)
+}
+
+func decodeCert(certData []byte) (*x509.Certificate, error) {
+	if len(certData) == 0 {
+		return nil, fmt.Errorf("missing cert data")
+	}
+	certificates, err := cert.ParseCertsPEM(certData)
+	if err != nil {
+		return nil, fmt.Errorf("bad certificate")
+	}
+
+	if len(certificates) == 0 {
+		return nil, fmt.Errorf("missing certificate")
+	}
+	clientCert := certificates[0]
+	if time.Now().After(clientCert.NotAfter) {
+		return nil, fmt.Errorf("already expired")
+	}
+	maxWait := clientCert.NotAfter.Sub(clientCert.NotBefore) / 5
+	latestTime := clientCert.NotAfter.Add(-maxWait)
+	if time.Now().After(latestTime) {
+		return nil, fmt.Errorf("expired in %6.3f seconds", clientCert.NotAfter.Sub(time.Now()).Seconds())
+	}
+
+	return clientCert, nil
+}
+
+func newCertSecret(secret *corev1.Secret, signerName string, caLifetime time.Duration) error {
+	if secret.Data == nil {
+		secret.Data = map[string][]byte{}
+	}
+
+	ca, err := crypto.MakeSelfSignedCAConfigForDuration(signerName, caLifetime)
+	if err != nil {
+		return err
+	}
+
+	caCertBytes := &bytes.Buffer{}
+	caKeyBytes := &bytes.Buffer{}
+	if err := ca.WriteCertConfig(caCertBytes, caKeyBytes); err != nil {
+		return err
+	}
+	secret.Data["ca.crt"] = caCertBytes.Bytes()
+
+	signingCertKeyPair, err := crypto.GetCAFromBytes(caCertBytes.Bytes(), caKeyBytes.Bytes())
+	if err != nil {
+		return err
+	}
+	clientUser := &user.DefaultInfo{
+		Name: signerName,
+	}
+	certConfig, err := signingCertKeyPair.MakeClientCertificateForDuration(clientUser, caLifetime)
+	if err != nil {
+		return err
+	}
+	certBytes := &bytes.Buffer{}
+	keyBytes := &bytes.Buffer{}
+	if err := certConfig.WriteCertConfig(certBytes, keyBytes); err != nil {
+		return err
+	}
+	secret.Data["tls.crt"] = certBytes.Bytes()
+	secret.Data["tls.key"] = keyBytes.Bytes()
+	return nil
+}
diff --git a/pkg/controllers/certrotation/certrotation_controller_test.go b/pkg/controllers/certrotation/certrotation_controller_test.go
new file mode 100644
index 000000000..85f61ecc2
--- /dev/null
+++ b/pkg/controllers/certrotation/certrotation_controller_test.go
@@ -0,0 +1,110 @@
+package certrotation
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	kubescheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"testing"
+	"time"
+)
+
+var (
+	scheme                 = runtime.NewScheme()
+	logCertSecretNamespace = "open-cluster-management"
+	logCertSecretName      = "ocm-klusterlet-self-signed-secrets"
+)
+
+func newTestReconciler(existingObjs ...runtime.Object) *Reconciler {
+	s := kubescheme.Scheme
+	s.AddKnownTypes(corev1.SchemeGroupVersion)
+	client := fake.NewFakeClientWithScheme(s, existingObjs...)
+	return &Reconciler{
+		client:                 client,
+		scheme:                 scheme,
+		logCertSecretNamespace: logCertSecretNamespace,
+		logCertSecretName:      logCertSecretName,
+	}
+}
+
+func newTestCertSecret(signerName string, caLifetime time.Duration) *corev1.Secret {
+	certSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: logCertSecretNamespace,
+			Name:      logCertSecretName,
+		},
+	}
+
+	newCertSecret(certSecret, signerName, caLifetime)
+	return certSecret
+}
+
+func TestReconciler(t *testing.T) {
+	ctx := context.TODO()
+	tests := []struct {
+		name           string
+		existingSecret *corev1.Secret
+		expectedCreate bool
+		expectedUpdate bool
+	}{
+		{
+			name: "no cert secret, create new one",
+			existingSecret: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
+				Namespace: "abc",
+				Name:      "abc"},
+			},
+			expectedCreate: true,
+			expectedUpdate: false,
+		},
+		{
+			name:           "has cert secret, not expire，  not create/update",
+			existingSecret: newTestCertSecret(signerName, defaultValidity),
+			expectedCreate: false,
+			expectedUpdate: false,
+		},
+		{
+			name:           "has cert secret, expire，update old one",
+			existingSecret: newTestCertSecret(signerName, -time.Minute*5),
+			expectedCreate: false,
+			expectedUpdate: true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			r := newTestReconciler(test.existingSecret)
+			res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "abc", Name: "abc"}})
+			if err != nil {
+				t.Errorf("unexpected error :%v", err)
+			}
+
+			assert.Equal(t, res, reconcile.Result{RequeueAfter: resyncInterval})
+
+			secret := &corev1.Secret{}
+			err = r.client.Get(ctx, types.NamespacedName{Name: logCertSecretName, Namespace: logCertSecretNamespace}, secret)
+			if err != nil {
+				t.Errorf("unexpected error :%v", err)
+			}
+
+			switch {
+			case test.expectedCreate:
+				if len(secret.Data) == 0 {
+					t.Errorf("should create cert secret")
+				}
+			case test.expectedUpdate:
+				if apiequality.Semantic.DeepEqual(secret.Data, test.existingSecret.Data) {
+					t.Errorf("should update cert secret")
+				}
+			default:
+				if !apiequality.Semantic.DeepEqual(secret.Data, test.existingSecret.Data) {
+					t.Errorf("should not update cert secret")
+				}
+			}
+		})
+	}
+}
diff --git a/pkg/controllers/clusterinfo/clusterinfo_controller.go b/pkg/controllers/clusterinfo/clusterinfo_controller.go
index de5598111..3665dadbf 100644
--- a/pkg/controllers/clusterinfo/clusterinfo_controller.go
+++ b/pkg/controllers/clusterinfo/clusterinfo_controller.go
@@ -2,6 +2,8 @@ package clusterinfo
 
 import (
 	"context"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/cache"
 	"reflect"
 
 	"github.com/open-cluster-management/multicloud-operators-foundation/pkg/utils"
@@ -30,31 +32,46 @@ const (
 	clusterFinalizerName = "managedclusterinfo.finalizers.open-cluster-management.io"
 )
 
+var (
+	logCertSecretNamespace string
+	logCertSecretName      string
+)
+
 type ClusterInfReconciler struct {
 	client client.Client
 	scheme *runtime.Scheme
-	caData []byte
 }
 
-func SetupWithManager(mgr manager.Manager, caData []byte) error {
-	if err := add(mgr, newClusterInfoReconciler(mgr, caData)); err != nil {
+func SetupWithManager(mgr manager.Manager, logCertSecret string) error {
+	var err error
+	logCertSecretNamespace, logCertSecretName, err = cache.SplitMetaNamespaceKey(logCertSecret)
+	if err != nil {
+		return err
+	}
+	if logCertSecretNamespace == "" {
+		logCertSecretNamespace, err = utils.GetComponentNamespace()
+		if err != nil {
+			return err
+		}
+	}
+	if err = add(mgr, newClusterInfoReconciler(mgr)); err != nil {
 		return err
 	}
-	if err := add(mgr, newAutoDetectReconciler(mgr)); err != nil {
+	if err = add(mgr, newAutoDetectReconciler(mgr)); err != nil {
 		return err
 	}
-	if err := add(mgr, newCapacityReconciler(mgr)); err != nil {
+	if err = add(mgr, newCapacityReconciler(mgr)); err != nil {
 		return err
 	}
 	return nil
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newClusterInfoReconciler(mgr manager.Manager, caData []byte) reconcile.Reconciler {
+func newClusterInfoReconciler(mgr manager.Manager) reconcile.Reconciler {
+
 	return &ClusterInfReconciler{
 		client: mgr.GetClient(),
 		scheme: mgr.GetScheme(),
-		caData: caData,
 	}
 }
 
@@ -76,6 +93,36 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 		return err
 	}
 
+	err = c.Watch(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(
+		handler.MapFunc(func(a client.Object) []reconcile.Request {
+			certSecret, ok := a.(*corev1.Secret)
+			if !ok {
+				// not a secret, returning empty
+				klog.Error("clusterinfo handler received non-secret object")
+				return []reconcile.Request{}
+			}
+
+			if certSecret.Name != logCertSecretName || certSecret.Namespace != logCertSecretNamespace {
+				return []reconcile.Request{}
+			}
+
+			managedClusterInfoList := &clusterinfov1beta1.ManagedClusterInfoList{}
+			err := mgr.GetClient().List(context.TODO(), managedClusterInfoList)
+			if err != nil {
+				klog.Error("Could not list managedClusterInfo", err)
+			}
+			var requests []reconcile.Request
+			for _, managedClusterInfo := range managedClusterInfoList.Items {
+				requests = append(requests, reconcile.Request{
+					NamespacedName: types.NamespacedName{
+						Name:      managedClusterInfo.Name,
+						Namespace: managedClusterInfo.Namespace,
+					},
+				})
+			}
+			return requests
+		}),
+	))
 	return nil
 }
 
@@ -117,15 +164,26 @@ func (r *ClusterInfReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, err
 	}
 
+	update := false
 	endpoint := ""
 	if len(cluster.Spec.ManagedClusterClientConfigs) != 0 {
 		endpoint = cluster.Spec.ManagedClusterClientConfigs[0].URL
+		if endpoint != "" && endpoint != clusterInfo.Spec.MasterEndpoint {
+			clusterInfo.Spec.MasterEndpoint = endpoint
+			update = true
+		}
+	}
+	caData, err := r.getLogCA()
+	if err != nil {
+		klog.Errorf("failed to get log CA. %v", err)
+	}
+
+	if caData != nil && !reflect.DeepEqual(caData, clusterInfo.Spec.LoggingCA) {
+		clusterInfo.Spec.LoggingCA = caData
+		update = true
 	}
 
-	if !reflect.DeepEqual(r.caData, clusterInfo.Spec.LoggingCA) ||
-		clusterInfo.Spec.MasterEndpoint != endpoint {
-		clusterInfo.Spec.LoggingCA = r.caData
-		clusterInfo.Spec.MasterEndpoint = endpoint
+	if update {
 		return ctrl.Result{}, r.client.Update(ctx, clusterInfo)
 	}
 
@@ -163,6 +221,11 @@ func (r *ClusterInfReconciler) newClusterInfoByManagedCluster(cluster *clusterv1
 		endpoint = cluster.Spec.ManagedClusterClientConfigs[0].URL
 	}
 
+	caData, err := r.getLogCA()
+	if err != nil {
+		klog.Errorf("failed to get log CA. %v", err)
+	}
+
 	return &clusterinfov1beta1.ManagedClusterInfo{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      cluster.Name,
@@ -171,7 +234,18 @@ func (r *ClusterInfReconciler) newClusterInfoByManagedCluster(cluster *clusterv1
 		},
 		Spec: clusterinfov1beta1.ClusterInfoSpec{
 			MasterEndpoint: endpoint,
-			LoggingCA:      r.caData,
+			LoggingCA:      caData,
 		},
 	}
 }
+
+func (r *ClusterInfReconciler) getLogCA() ([]byte, error) {
+	logCertSecret := &corev1.Secret{}
+	err := r.client.Get(context.TODO(), types.NamespacedName{Name: logCertSecretName, Namespace: logCertSecretNamespace}, logCertSecret)
+	if err != nil {
+		klog.Errorf("failed to get log cert secret %v/%v: %v", logCertSecretNamespace, logCertSecretName, err)
+		return nil, err
+	}
+	caData := logCertSecret.Data["ca.crt"]
+	return caData, nil
+}
diff --git a/pkg/controllers/clusterinfo/clusterinfo_controller_test.go b/pkg/controllers/clusterinfo/clusterinfo_controller_test.go
index 8e18a0624..79363b45f 100644
--- a/pkg/controllers/clusterinfo/clusterinfo_controller_test.go
+++ b/pkg/controllers/clusterinfo/clusterinfo_controller_test.go
@@ -2,6 +2,7 @@ package clusterinfo
 
 import (
 	"context"
+	corev1 "k8s.io/api/core/v1"
 	"os"
 	"path/filepath"
 	"sync"
@@ -43,6 +44,7 @@ func TestMain(m *testing.M) {
 
 	clusterinfov1beta1.AddToScheme(cliScheme.Scheme)
 	clusterv1.AddToScheme(cliScheme.Scheme)
+	corev1.AddToScheme(cliScheme.Scheme)
 
 	var err error
 	if cfg, err = t.Start(); err != nil {
@@ -52,7 +54,7 @@ func TestMain(m *testing.M) {
 	// AddToSchemes may be used to add all resources defined in the project to a Scheme
 	var AddToSchemes runtime.SchemeBuilder
 	// Register the types with the Scheme so the components can map objects to GroupVersionKinds and back
-	AddToSchemes = append(AddToSchemes, clusterv1.Install, clusterinfov1beta1.AddToScheme)
+	AddToSchemes = append(AddToSchemes, clusterv1.Install, clusterinfov1beta1.AddToScheme, corev1.AddToScheme)
 
 	if err := AddToSchemes.AddToScheme(scheme); err != nil {
 		klog.Errorf("Failed adding apis to scheme, %v", err)
@@ -67,6 +69,10 @@ func TestMain(m *testing.M) {
 		klog.Errorf("Failed adding cluster to scheme, %v", err)
 		os.Exit(1)
 	}
+	if err := corev1.AddToScheme(scheme); err != nil {
+		klog.Errorf("Failed adding core v1 to scheme, %v", err)
+		os.Exit(1)
+	}
 
 	exitVal := m.Run()
 	os.Exit(exitVal)
@@ -96,7 +102,7 @@ func TestControllerReconcile(t *testing.T) {
 
 	c = mgr.GetClient()
 
-	SetupWithManager(mgr, []byte(""))
+	SetupWithManager(mgr, "open-cluster-management/ocm-klusterlet-self-signed-secrets")
 
 	cancel, mgrStopped := StartTestManager(mgr, g)
 
@@ -120,7 +126,6 @@ func newTestClusterInfoReconciler(existingObjs []runtime.Object) *ClusterInfReco
 	return &ClusterInfReconciler{
 		client: fake.NewFakeClientWithScheme(scheme, existingObjs...),
 		scheme: scheme,
-		caData: []byte{},
 	}
 }
 
@@ -237,3 +242,47 @@ func TestReconcile(t *testing.T) {
 		})
 	}
 }
+
+func Test_getLogCA(t *testing.T) {
+	logCertSecretNamespace = "open-cluster-management"
+	logCertSecretName = "ocm-klusterlet-self-signed-secrets"
+
+	tests := []struct {
+		name         string
+		existingObjs []runtime.Object
+		expectCAData []byte
+	}{
+		{
+			name: "get log ca",
+			existingObjs: []runtime.Object{&corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: logCertSecretName, Namespace: logCertSecretNamespace},
+				Data: map[string][]byte{
+					"ca.crt": {123},
+				},
+			}},
+			expectCAData: []byte{123},
+		},
+		{
+			name:         "no cert secret",
+			existingObjs: []runtime.Object{},
+			expectCAData: nil,
+		},
+		{
+			name: "no ca data",
+			existingObjs: []runtime.Object{&corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: logCertSecretName, Namespace: logCertSecretNamespace},
+				Data:       nil,
+			}},
+			expectCAData: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			svrc := newTestClusterInfoReconciler(test.existingObjs)
+			caData, _ := svrc.getLogCA()
+
+			assert.Equal(t, test.expectCAData, caData)
+		})
+	}
+}
diff --git a/pkg/klusterlet/agent/agent.go b/pkg/klusterlet/agent/agent.go
index 052eada02..5070723a7 100644
--- a/pkg/klusterlet/agent/agent.go
+++ b/pkg/klusterlet/agent/agent.go
@@ -83,7 +83,7 @@ func (k *Agent) RefreshServerIfNeeded(clusterInfo *v1beta1.ManagedClusterInfo) {
 	if k.server.isShutDown() {
 		return
 	}
-
+	klog.Infof("refresh server: %v\n", caData)
 	go k.server.refresh(caData, pool)
 }
 
diff --git a/pkg/klusterlet/clusterinfo/clusterinfo_controller.go b/pkg/klusterlet/clusterinfo/clusterinfo_controller.go
index 23bb137aa..b2e83f0cb 100644
--- a/pkg/klusterlet/clusterinfo/clusterinfo_controller.go
+++ b/pkg/klusterlet/clusterinfo/clusterinfo_controller.go
@@ -72,6 +72,9 @@ func (r *ClusterInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	// refresh logging server CA if CA is changed
+	r.RefreshAgentServer(clusterInfo)
+
 	// Update cluster info status here.
 	newStatus := clusterInfo.DeepCopy().Status
 	var errs []error
@@ -155,8 +158,6 @@ func (r *ClusterInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, err
 	}
 
-	r.RefreshAgentServer(clusterInfo)
-
 	return ctrl.Result{}, nil
 }
 
@@ -419,7 +420,7 @@ func ocpDistributionInfoUpdated(old, new *clusterv1beta1.OCPDistributionInfo) bo
 func (r *ClusterInfoReconciler) RefreshAgentServer(clusterInfo *clusterv1beta1.ManagedClusterInfo) {
 	select {
 	case r.Agent.RunServer <- *clusterInfo:
-		log.Info("Signal agent server to start")
+		klog.Info("Signal agent server to start")
 	default:
 	}
 
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index 710bc00c8..0c7dfe0b8 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -46,6 +46,7 @@ var (
 	managedClusterName     string
 	managedClusterSetName  = "clusterset1"
 	fakeManagedClusterName = util.RandomName()
+	foundationNS           string
 )
 
 // This suite is sensitive to the following environment variables:
@@ -60,6 +61,11 @@ var _ = ginkgo.BeforeSuite(func() {
 		managedClusterName = "cluster1"
 	}
 
+	foundationNS = os.Getenv("FOUNDATION_NS")
+	if foundationNS == "" {
+		foundationNS = "open-cluster-management"
+	}
+
 	dynamicClient, err = util.NewDynamicClient()
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
@@ -79,10 +85,6 @@ var _ = ginkgo.BeforeSuite(func() {
 	clusterClient, err = clusterclient.NewForConfig(cfg)
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
-	gomega.Eventually(func() error {
-		return util.CheckFoundationPodsReady()
-	}, eventuallyTimeout, 2*eventuallyInterval).Should(gomega.Succeed())
-
 	// accept the managed cluster that is deployed by registration-operator
 	err = util.CheckJoinedManagedCluster(clusterClient, managedClusterName)
 	if err != nil {
diff --git a/test/e2e/log_test.go b/test/e2e/log_test.go
index 445af75c0..e2f631c9e 100644
--- a/test/e2e/log_test.go
+++ b/test/e2e/log_test.go
@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/onsi/ginkgo"
@@ -68,35 +69,42 @@ var _ = ginkgo.Describe("Testing Pod log", func() {
 	})
 
 	ginkgo.It("should get log from pod successfully", func() {
-		gomega.Eventually(func() bool {
-			exists, err := util.HasResource(dynamicClient, clusterInfoGVR, managedClusterName, managedClusterName)
-			if err != nil {
-				return false
-			}
-			return exists
-		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
-
-		gomega.Eventually(func() bool {
+		// check the ManagedClusterInfo status
+		gomega.Eventually(func() error {
 			managedClusterInfo, err := util.GetResource(dynamicClient, clusterInfoGVR, managedClusterName, managedClusterName)
 			if err != nil {
-				return false
+				return err
 			}
-			// check the ManagedClusterInfo status
-			return util.GetConditionTypeFromStatus(managedClusterInfo, clusterinfov1beta1.ManagedClusterInfoSynced)
-		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+			if !util.GetConditionTypeFromStatus(managedClusterInfo, clusterinfov1beta1.ManagedClusterInfoSynced) {
+				return fmt.Errorf("the condition of managedClusterInfo is not synced")
+			}
+			return nil
+		}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
 
-		req := restClient.Get().Namespace(managedClusterName).
-			Name(managedClusterName).
-			Resource("clusterstatuses").
-			SubResource("log").Suffix(podNamespace, podName, containerName)
+		// case1: get logs successfully
+		gomega.Eventually(func() error {
+			req := restClient.Get().Namespace(managedClusterName).
+				Name(managedClusterName).
+				Resource("clusterstatuses").
+				SubResource("log").Suffix(podNamespace, podName, containerName)
 
-		gomega.Eventually(func() bool {
 			_, err := req.DoRaw(context.TODO())
+			return err
+		}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
 
-			if err != nil {
-				return false
-			}
-			return true
-		}, 60*time.Second, 1*time.Second).Should(gomega.BeTrue())
+		// case2: get logs successfully after cert rotation
+		gomega.Eventually(func() error {
+			return kubeClient.CoreV1().Secrets(foundationNS).Delete(context.TODO(), "ocm-klusterlet-self-signed-secrets", metav1.DeleteOptions{})
+		}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
+
+		gomega.Eventually(func() error {
+			req := restClient.Get().Namespace(managedClusterName).
+				Name(managedClusterName).
+				Resource("clusterstatuses").
+				SubResource("log").Suffix(podNamespace, podName, containerName)
+
+			_, err := req.DoRaw(context.TODO())
+			return err
+		}, eventuallyTimeout*2, eventuallyInterval*5).ShouldNot(gomega.HaveOccurred())
 	})
 })
diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go
index 054a5b193..fb36d12aa 100644
--- a/test/e2e/util/util.go
+++ b/test/e2e/util/util.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	clusterinfov1beta1 "github.com/open-cluster-management/multicloud-operators-foundation/pkg/apis/internal.open-cluster-management.io/v1beta1"
 	hiveclient "github.com/openshift/hive/pkg/client/clientset/versioned"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/rand"
 	"os"
 	"os/user"
@@ -341,66 +340,6 @@ func CheckClusterID(obj *unstructured.Unstructured) error {
 	return nil
 }
 
-func CheckFoundationPodsReady() error {
-	clusterCfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
-	if err != nil {
-		return err
-	}
-
-	hubClient, err := kubernetes.NewForConfig(clusterCfg)
-	if err != nil {
-		return err
-	}
-
-	_, err = hubClient.CoreV1().Secrets("open-cluster-management").Get(context.TODO(),
-		"foundation-webhook", metav1.GetOptions{})
-	if err != nil {
-		return fmt.Errorf("failed to get secret foundation-webhook; %v", err)
-	}
-
-	labelNs := map[string]string{
-		"app=foundation-controller":  "open-cluster-management",
-		"app=foundation-proxyserver": "open-cluster-management",
-		"app=foundation-webhook":     "open-cluster-management",
-		"app=work-manager":           "open-cluster-management-agent",
-	}
-	for label, ns := range labelNs {
-		pods, err := hubClient.CoreV1().Pods(ns).List(context.TODO(),
-			metav1.ListOptions{LabelSelector: label})
-		if err != nil {
-			return fmt.Errorf("failed to get %v pods. %#v", label, err)
-		}
-		if len(pods.Items) == 0 {
-			return fmt.Errorf("failed to get %v pods", label)
-		}
-
-		for _, pod := range pods.Items {
-			if err := podConditionsReady(pod); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func podConditionsReady(pod corev1.Pod) error {
-	if len(pod.Status.Conditions) == 0 {
-		return fmt.Errorf("the pod %v conditions is null", pod.Name)
-	}
-
-	for _, condition := range pod.Status.Conditions {
-		if condition.Type == corev1.PodReady && condition.Status != corev1.ConditionTrue {
-			return fmt.Errorf("the pod %v conditions is not ready", pod.Name)
-		}
-		if condition.Type == corev1.ContainersReady && condition.Status != corev1.ConditionTrue {
-			return fmt.Errorf("the containers of pod %v conditions are not ready", pod.Name)
-		}
-	}
-
-	return nil
-}
-
 func RandomName() string {
 	return "test-automation-" + rand.String(6)
 }
diff --git a/test/e2e/webhook_test.go b/test/e2e/webhook_test.go
index 6d72ac932..41b38242e 100644
--- a/test/e2e/webhook_test.go
+++ b/test/e2e/webhook_test.go
@@ -218,11 +218,11 @@ var _ = ginkgo.Describe("Testing webhook cert rotation", func() {
 			return err
 		}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
 
-		err = kubeClient.CoreV1().Secrets("open-cluster-management").Delete(context.TODO(), "foundation-webhook", metav1.DeleteOptions{})
+		err = kubeClient.CoreV1().Secrets(foundationNS).Delete(context.TODO(), "foundation-webhook", metav1.DeleteOptions{})
 		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 
 		gomega.Eventually(func() error {
-			_, err := kubeClient.CoreV1().Secrets("open-cluster-management").Get(context.TODO(), "foundation-webhook", metav1.GetOptions{})
+			_, err := kubeClient.CoreV1().Secrets(foundationNS).Get(context.TODO(), "foundation-webhook", metav1.GetOptions{})
 			return err
 		}, eventuallyTimeout, eventuallyInterval*5).ShouldNot(gomega.HaveOccurred())
 
diff --git a/vendor/github.com/openshift/library-go/pkg/crypto/crypto.go b/vendor/github.com/openshift/library-go/pkg/crypto/crypto.go
new file mode 100644
index 000000000..544ea9fb8
--- /dev/null
+++ b/vendor/github.com/openshift/library-go/pkg/crypto/crypto.go
@@ -0,0 +1,1182 @@
+package crypto
+
+import (
+	"bytes"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/big"
+	mathrand "math/rand"
+	"net"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/client-go/util/cert"
+)
+
+// TLS versions that are known to golang. Go 1.13 adds support for
+// TLS 1.3 that's opt-out with a build flag.
+var versions = map[string]uint16{
+	"VersionTLS10": tls.VersionTLS10,
+	"VersionTLS11": tls.VersionTLS11,
+	"VersionTLS12": tls.VersionTLS12,
+	"VersionTLS13": tls.VersionTLS13,
+}
+
+// TLS versions that are enabled.
+var supportedVersions = map[string]uint16{
+	"VersionTLS10": tls.VersionTLS10,
+	"VersionTLS11": tls.VersionTLS11,
+	"VersionTLS12": tls.VersionTLS12,
+	"VersionTLS13": tls.VersionTLS13,
+}
+
+// TLSVersionToNameOrDie given a tls version as an int, return its readable name
+func TLSVersionToNameOrDie(intVal uint16) string {
+	matches := []string{}
+	for key, version := range versions {
+		if version == intVal {
+			matches = append(matches, key)
+		}
+	}
+
+	if len(matches) == 0 {
+		panic(fmt.Sprintf("no name found for %d", intVal))
+	}
+	if len(matches) > 1 {
+		panic(fmt.Sprintf("multiple names found for %d: %v", intVal, matches))
+	}
+	return matches[0]
+}
+
+func TLSVersion(versionName string) (uint16, error) {
+	if len(versionName) == 0 {
+		return DefaultTLSVersion(), nil
+	}
+	if version, ok := versions[versionName]; ok {
+		return version, nil
+	}
+	return 0, fmt.Errorf("unknown tls version %q", versionName)
+}
+func TLSVersionOrDie(versionName string) uint16 {
+	version, err := TLSVersion(versionName)
+	if err != nil {
+		panic(err)
+	}
+	return version
+}
+
+// TLS versions that are known to golang, but may not necessarily be enabled.
+func GolangTLSVersions() []string {
+	supported := []string{}
+	for k := range versions {
+		supported = append(supported, k)
+	}
+	sort.Strings(supported)
+	return supported
+}
+
+// Returns the build enabled TLS versions.
+func ValidTLSVersions() []string {
+	validVersions := []string{}
+	for k := range supportedVersions {
+		validVersions = append(validVersions, k)
+	}
+	sort.Strings(validVersions)
+	return validVersions
+}
+func DefaultTLSVersion() uint16 {
+	// Can't use SSLv3 because of POODLE and BEAST
+	// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+	// Can't use TLSv1.1 because of RC4 cipher usage
+	return tls.VersionTLS12
+}
+
+// ciphersTLS13 copies golang 1.13 implementation, where TLS1.3 suites are not
+// configurable (cipherSuites field is ignored for TLS1.3 flows and all of the
+// below three - and none other - are used)
+var ciphersTLS13 = map[string]uint16{
+	"TLS_AES_128_GCM_SHA256":       tls.TLS_AES_128_GCM_SHA256,
+	"TLS_AES_256_GCM_SHA384":       tls.TLS_AES_256_GCM_SHA384,
+	"TLS_CHACHA20_POLY1305_SHA256": tls.TLS_CHACHA20_POLY1305_SHA256,
+}
+
+var ciphers = map[string]uint16{
+	"TLS_RSA_WITH_RC4_128_SHA":                      tls.TLS_RSA_WITH_RC4_128_SHA,
+	"TLS_RSA_WITH_3DES_EDE_CBC_SHA":                 tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
+	"TLS_RSA_WITH_AES_128_CBC_SHA":                  tls.TLS_RSA_WITH_AES_128_CBC_SHA,
+	"TLS_RSA_WITH_AES_256_CBC_SHA":                  tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+	"TLS_RSA_WITH_AES_128_CBC_SHA256":               tls.TLS_RSA_WITH_AES_128_CBC_SHA256,
+	"TLS_RSA_WITH_AES_128_GCM_SHA256":               tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
+	"TLS_RSA_WITH_AES_256_GCM_SHA384":               tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":              tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":          tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":          tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_RC4_128_SHA":                tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
+	"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA":           tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":            tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":            tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256":       tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256":         tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":         tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256":       tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384":         tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384":       tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305":          tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+	"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305":        tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
+	"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256":   tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+	"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
+}
+
+// openSSLToIANACiphersMap maps OpenSSL cipher suite names to IANA names
+// ref: https://www.iana.org/assignments/tls-parameters/tls-parameters.xml
+var openSSLToIANACiphersMap = map[string]string{
+	// TLS 1.3 ciphers - not configurable in go 1.13, all of them are used in TLSv1.3 flows
+	//	"TLS_AES_128_GCM_SHA256":       "TLS_AES_128_GCM_SHA256",       // 0x13,0x01
+	//	"TLS_AES_256_GCM_SHA384":       "TLS_AES_256_GCM_SHA384",       // 0x13,0x02
+	//	"TLS_CHACHA20_POLY1305_SHA256": "TLS_CHACHA20_POLY1305_SHA256", // 0x13,0x03
+
+	// TLS 1.2
+	"ECDHE-ECDSA-AES128-GCM-SHA256": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",       // 0xC0,0x2B
+	"ECDHE-RSA-AES128-GCM-SHA256":   "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",         // 0xC0,0x2F
+	"ECDHE-ECDSA-AES256-GCM-SHA384": "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",       // 0xC0,0x2C
+	"ECDHE-RSA-AES256-GCM-SHA384":   "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",         // 0xC0,0x30
+	"ECDHE-ECDSA-CHACHA20-POLY1305": "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", // 0xCC,0xA9
+	"ECDHE-RSA-CHACHA20-POLY1305":   "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",   // 0xCC,0xA8
+	"ECDHE-ECDSA-AES128-SHA256":     "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",       // 0xC0,0x23
+	"ECDHE-RSA-AES128-SHA256":       "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",         // 0xC0,0x27
+	"AES128-GCM-SHA256":             "TLS_RSA_WITH_AES_128_GCM_SHA256",               // 0x00,0x9C
+	"AES256-GCM-SHA384":             "TLS_RSA_WITH_AES_256_GCM_SHA384",               // 0x00,0x9D
+	"AES128-SHA256":                 "TLS_RSA_WITH_AES_128_CBC_SHA256",               // 0x00,0x3C
+
+	// TLS 1
+	"ECDHE-ECDSA-AES128-SHA": "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", // 0xC0,0x09
+	"ECDHE-RSA-AES128-SHA":   "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",   // 0xC0,0x13
+	"ECDHE-ECDSA-AES256-SHA": "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", // 0xC0,0x0A
+	"ECDHE-RSA-AES256-SHA":   "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",   // 0xC0,0x14
+
+	// SSL 3
+	"AES128-SHA":   "TLS_RSA_WITH_AES_128_CBC_SHA",  // 0x00,0x2F
+	"AES256-SHA":   "TLS_RSA_WITH_AES_256_CBC_SHA",  // 0x00,0x35
+	"DES-CBC3-SHA": "TLS_RSA_WITH_3DES_EDE_CBC_SHA", // 0x00,0x0A
+}
+
+// CipherSuitesToNamesOrDie given a list of cipher suites as ints, return their readable names
+func CipherSuitesToNamesOrDie(intVals []uint16) []string {
+	ret := []string{}
+	for _, intVal := range intVals {
+		ret = append(ret, CipherSuiteToNameOrDie(intVal))
+	}
+
+	return ret
+}
+
+// CipherSuiteToNameOrDie given a cipher suite as an int, return its readable name
+func CipherSuiteToNameOrDie(intVal uint16) string {
+	// The following suite ids appear twice in the cipher map (with
+	// and without the _SHA256 suffix) for the purposes of backwards
+	// compatibility. Always return the current rather than the legacy
+	// name.
+	switch intVal {
+	case tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:
+		return "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"
+	case tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:
+		return "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"
+	}
+
+	matches := []string{}
+	for key, version := range ciphers {
+		if version == intVal {
+			matches = append(matches, key)
+		}
+	}
+
+	if len(matches) == 0 {
+		panic(fmt.Sprintf("no name found for %d", intVal))
+	}
+	if len(matches) > 1 {
+		panic(fmt.Sprintf("multiple names found for %d: %v", intVal, matches))
+	}
+	return matches[0]
+}
+
+func CipherSuite(cipherName string) (uint16, error) {
+	if cipher, ok := ciphers[cipherName]; ok {
+		return cipher, nil
+	}
+
+	if _, ok := ciphersTLS13[cipherName]; ok {
+		return 0, fmt.Errorf("all golang TLSv1.3 ciphers are always used for TLSv1.3 flows")
+	}
+
+	return 0, fmt.Errorf("unknown cipher name %q", cipherName)
+}
+
+func CipherSuitesOrDie(cipherNames []string) []uint16 {
+	if len(cipherNames) == 0 {
+		return DefaultCiphers()
+	}
+	cipherValues := []uint16{}
+	for _, cipherName := range cipherNames {
+		cipher, err := CipherSuite(cipherName)
+		if err != nil {
+			panic(err)
+		}
+		cipherValues = append(cipherValues, cipher)
+	}
+	return cipherValues
+}
+func ValidCipherSuites() []string {
+	validCipherSuites := []string{}
+	for k := range ciphers {
+		validCipherSuites = append(validCipherSuites, k)
+	}
+	sort.Strings(validCipherSuites)
+	return validCipherSuites
+}
+func DefaultCiphers() []uint16 {
+	// HTTP/2 mandates TLS 1.2 or higher with an AEAD cipher
+	// suite (GCM, Poly1305) and ephemeral key exchange (ECDHE, DHE) for
+	// perfect forward secrecy. Servers may provide additional cipher
+	// suites for backwards compatibility with HTTP/1.1 clients.
+	// See RFC7540, section 9.2 (Use of TLS Features) and Appendix A
+	// (TLS 1.2 Cipher Suite Black List).
+	return []uint16{
+		tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
+		tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // required by http/2
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // forbidden by http/2, not flagged by http2isBadCipher() in go1.8
+		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,   // forbidden by http/2, not flagged by http2isBadCipher() in go1.8
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,    // forbidden by http/2
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,    // forbidden by http/2
+		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,      // forbidden by http/2
+		tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,      // forbidden by http/2
+		tls.TLS_RSA_WITH_AES_128_GCM_SHA256,         // forbidden by http/2
+		tls.TLS_RSA_WITH_AES_256_GCM_SHA384,         // forbidden by http/2
+		// the next one is in the intermediate suite, but go1.8 http2isBadCipher() complains when it is included at the recommended index
+		// because it comes after ciphers forbidden by the http/2 spec
+		// tls.TLS_RSA_WITH_AES_128_CBC_SHA256,
+		// tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, // forbidden by http/2, disabled to mitigate SWEET32 attack
+		// tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,       // forbidden by http/2, disabled to mitigate SWEET32 attack
+		tls.TLS_RSA_WITH_AES_128_CBC_SHA, // forbidden by http/2
+		tls.TLS_RSA_WITH_AES_256_CBC_SHA, // forbidden by http/2
+	}
+}
+
+// SecureTLSConfig enforces the default minimum security settings for the cluster.
+func SecureTLSConfig(config *tls.Config) *tls.Config {
+	if config.MinVersion == 0 {
+		config.MinVersion = DefaultTLSVersion()
+	}
+
+	config.PreferServerCipherSuites = true
+	if len(config.CipherSuites) == 0 {
+		config.CipherSuites = DefaultCiphers()
+	}
+	return config
+}
+
+// OpenSSLToIANACipherSuites maps input OpenSSL Cipher Suite names to their
+// IANA counterparts.
+// Unknown ciphers are left out.
+func OpenSSLToIANACipherSuites(ciphers []string) []string {
+	ianaCiphers := make([]string, 0, len(ciphers))
+
+	for _, c := range ciphers {
+		ianaCipher, found := openSSLToIANACiphersMap[c]
+		if found {
+			ianaCiphers = append(ianaCiphers, ianaCipher)
+		}
+	}
+
+	return ianaCiphers
+}
+
+type TLSCertificateConfig struct {
+	Certs []*x509.Certificate
+	Key   crypto.PrivateKey
+}
+
+type TLSCARoots struct {
+	Roots []*x509.Certificate
+}
+
+func (c *TLSCertificateConfig) WriteCertConfigFile(certFile, keyFile string) error {
+	// ensure parent dir
+	if err := os.MkdirAll(filepath.Dir(certFile), os.FileMode(0755)); err != nil {
+		return err
+	}
+	certFileWriter, err := os.OpenFile(certFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(keyFile), os.FileMode(0755)); err != nil {
+		return err
+	}
+	keyFileWriter, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+
+	if err := writeCertificates(certFileWriter, c.Certs...); err != nil {
+		return err
+	}
+	if err := writeKeyFile(keyFileWriter, c.Key); err != nil {
+		return err
+	}
+
+	if err := certFileWriter.Close(); err != nil {
+		return err
+	}
+	if err := keyFileWriter.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *TLSCertificateConfig) WriteCertConfig(certFile, keyFile io.Writer) error {
+	if err := writeCertificates(certFile, c.Certs...); err != nil {
+		return err
+	}
+	if err := writeKeyFile(keyFile, c.Key); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *TLSCertificateConfig) GetPEMBytes() ([]byte, []byte, error) {
+	certBytes, err := EncodeCertificates(c.Certs...)
+	if err != nil {
+		return nil, nil, err
+	}
+	keyBytes, err := encodeKey(c.Key)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return certBytes, keyBytes, nil
+}
+
+func GetTLSCertificateConfig(certFile, keyFile string) (*TLSCertificateConfig, error) {
+	if len(certFile) == 0 {
+		return nil, errors.New("certFile missing")
+	}
+	if len(keyFile) == 0 {
+		return nil, errors.New("keyFile missing")
+	}
+
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	certs, err := cert.ParseCertsPEM(certPEMBlock)
+	if err != nil {
+		return nil, fmt.Errorf("Error reading %s: %s", certFile, err)
+	}
+
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+	keyPairCert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	if err != nil {
+		return nil, err
+	}
+	key := keyPairCert.PrivateKey
+
+	return &TLSCertificateConfig{certs, key}, nil
+}
+
+func GetTLSCertificateConfigFromBytes(certBytes, keyBytes []byte) (*TLSCertificateConfig, error) {
+	if len(certBytes) == 0 {
+		return nil, errors.New("certFile missing")
+	}
+	if len(keyBytes) == 0 {
+		return nil, errors.New("keyFile missing")
+	}
+
+	certs, err := cert.ParseCertsPEM(certBytes)
+	if err != nil {
+		return nil, fmt.Errorf("Error reading cert: %s", err)
+	}
+
+	keyPairCert, err := tls.X509KeyPair(certBytes, keyBytes)
+	if err != nil {
+		return nil, err
+	}
+	key := keyPairCert.PrivateKey
+
+	return &TLSCertificateConfig{certs, key}, nil
+}
+
+const (
+	DefaultCertificateLifetimeInDays   = 365 * 2 // 2 years
+	DefaultCACertificateLifetimeInDays = 365 * 5 // 5 years
+
+	// Default keys are 2048 bits
+	keyBits = 2048
+)
+
+type CA struct {
+	Config *TLSCertificateConfig
+
+	SerialGenerator SerialGenerator
+}
+
+// SerialGenerator is an interface for getting a serial number for the cert.  It MUST be thread-safe.
+type SerialGenerator interface {
+	Next(template *x509.Certificate) (int64, error)
+}
+
+// SerialFileGenerator returns a unique, monotonically increasing serial number and ensures the CA on disk records that value.
+type SerialFileGenerator struct {
+	SerialFile string
+
+	// lock guards access to the Serial field
+	lock   sync.Mutex
+	Serial int64
+}
+
+func NewSerialFileGenerator(serialFile string) (*SerialFileGenerator, error) {
+	// read serial file, it must already exist
+	serial, err := fileToSerial(serialFile)
+	if err != nil {
+		return nil, err
+	}
+
+	generator := &SerialFileGenerator{
+		Serial:     serial,
+		SerialFile: serialFile,
+	}
+
+	// 0 is unused and 1 is reserved for the CA itself
+	// Thus we need to guarantee that the first external call to SerialFileGenerator.Next returns 2+
+	// meaning that SerialFileGenerator.Serial must not be less than 1 (it is guaranteed to be non-negative)
+	if generator.Serial < 1 {
+		// fake a call to Next so the file stays in sync and Serial is incremented
+		if _, err := generator.Next(&x509.Certificate{}); err != nil {
+			return nil, err
+		}
+	}
+
+	return generator, nil
+}
+
+// Next returns a unique, monotonically increasing serial number and ensures the CA on disk records that value.
+func (s *SerialFileGenerator) Next(template *x509.Certificate) (int64, error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	// do a best effort check to make sure concurrent external writes are not occurring to the underlying serial file
+	serial, err := fileToSerial(s.SerialFile)
+	if err != nil {
+		return 0, err
+	}
+	if serial != s.Serial {
+		return 0, fmt.Errorf("serial file %s out of sync ram=%d disk=%d", s.SerialFile, s.Serial, serial)
+	}
+
+	next := s.Serial + 1
+	s.Serial = next
+
+	// Output in hex, padded to multiples of two characters for OpenSSL's sake
+	serialText := fmt.Sprintf("%X", next)
+	if len(serialText)%2 == 1 {
+		serialText = "0" + serialText
+	}
+	// always add a newline at the end to have a valid file
+	serialText += "\n"
+
+	if err := ioutil.WriteFile(s.SerialFile, []byte(serialText), os.FileMode(0640)); err != nil {
+		return 0, err
+	}
+	return next, nil
+}
+
+func fileToSerial(serialFile string) (int64, error) {
+	serialData, err := ioutil.ReadFile(serialFile)
+	if err != nil {
+		return 0, err
+	}
+
+	// read the file as a single hex number after stripping any whitespace
+	serial, err := strconv.ParseInt(string(bytes.TrimSpace(serialData)), 16, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	if serial < 0 {
+		return 0, fmt.Errorf("invalid negative serial %d in serial file %s", serial, serialFile)
+	}
+
+	return serial, nil
+}
+
+// RandomSerialGenerator returns a serial based on time.Now and the subject
+type RandomSerialGenerator struct {
+}
+
+func (s *RandomSerialGenerator) Next(template *x509.Certificate) (int64, error) {
+	return randomSerialNumber(), nil
+}
+
+// randomSerialNumber returns a random int64 serial number based on
+// time.Now. It is defined separately from the generator interface so
+// that the caller doesn't have to worry about an input template or
+// error - these are unnecessary when creating a random serial.
+func randomSerialNumber() int64 {
+	r := mathrand.New(mathrand.NewSource(time.Now().UTC().UnixNano()))
+	return r.Int63()
+}
+
+// EnsureCA returns a CA, whether it was created (as opposed to pre-existing), and any error
+// if serialFile is empty, a RandomSerialGenerator will be used
+func EnsureCA(certFile, keyFile, serialFile, name string, expireDays int) (*CA, bool, error) {
+	if ca, err := GetCA(certFile, keyFile, serialFile); err == nil {
+		return ca, false, err
+	}
+	ca, err := MakeSelfSignedCA(certFile, keyFile, serialFile, name, expireDays)
+	return ca, true, err
+}
+
+// if serialFile is empty, a RandomSerialGenerator will be used
+func GetCA(certFile, keyFile, serialFile string) (*CA, error) {
+	caConfig, err := GetTLSCertificateConfig(certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	var serialGenerator SerialGenerator
+	if len(serialFile) > 0 {
+		serialGenerator, err = NewSerialFileGenerator(serialFile)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		serialGenerator = &RandomSerialGenerator{}
+	}
+
+	return &CA{
+		SerialGenerator: serialGenerator,
+		Config:          caConfig,
+	}, nil
+}
+
+func GetCAFromBytes(certBytes, keyBytes []byte) (*CA, error) {
+	caConfig, err := GetTLSCertificateConfigFromBytes(certBytes, keyBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	return &CA{
+		SerialGenerator: &RandomSerialGenerator{},
+		Config:          caConfig,
+	}, nil
+}
+
+// if serialFile is empty, a RandomSerialGenerator will be used
+func MakeSelfSignedCA(certFile, keyFile, serialFile, name string, expireDays int) (*CA, error) {
+	klog.V(2).Infof("Generating new CA for %s cert, and key in %s, %s", name, certFile, keyFile)
+
+	caConfig, err := MakeSelfSignedCAConfig(name, expireDays)
+	if err != nil {
+		return nil, err
+	}
+	if err := caConfig.WriteCertConfigFile(certFile, keyFile); err != nil {
+		return nil, err
+	}
+
+	var serialGenerator SerialGenerator
+	if len(serialFile) > 0 {
+		// create / overwrite the serial file with a zero padded hex value (ending in a newline to have a valid file)
+		if err := ioutil.WriteFile(serialFile, []byte("00\n"), 0644); err != nil {
+			return nil, err
+		}
+		serialGenerator, err = NewSerialFileGenerator(serialFile)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		serialGenerator = &RandomSerialGenerator{}
+	}
+
+	return &CA{
+		SerialGenerator: serialGenerator,
+		Config:          caConfig,
+	}, nil
+}
+
+func MakeSelfSignedCAConfig(name string, expireDays int) (*TLSCertificateConfig, error) {
+	subject := pkix.Name{CommonName: name}
+	return MakeSelfSignedCAConfigForSubject(subject, expireDays)
+}
+
+func MakeSelfSignedCAConfigForSubject(subject pkix.Name, expireDays int) (*TLSCertificateConfig, error) {
+	var caLifetimeInDays = DefaultCACertificateLifetimeInDays
+	if expireDays > 0 {
+		caLifetimeInDays = expireDays
+	}
+
+	if caLifetimeInDays > DefaultCACertificateLifetimeInDays {
+		warnAboutCertificateLifeTime(subject.CommonName, DefaultCACertificateLifetimeInDays)
+	}
+
+	caLifetime := time.Duration(caLifetimeInDays) * 24 * time.Hour
+	return makeSelfSignedCAConfigForSubjectAndDuration(subject, caLifetime)
+}
+
+func MakeSelfSignedCAConfigForDuration(name string, caLifetime time.Duration) (*TLSCertificateConfig, error) {
+	subject := pkix.Name{CommonName: name}
+	return makeSelfSignedCAConfigForSubjectAndDuration(subject, caLifetime)
+}
+
+func makeSelfSignedCAConfigForSubjectAndDuration(subject pkix.Name, caLifetime time.Duration) (*TLSCertificateConfig, error) {
+	// Create CA cert
+	rootcaPublicKey, rootcaPrivateKey, publicKeyHash, err := newKeyPairWithHash()
+	if err != nil {
+		return nil, err
+	}
+	// AuthorityKeyId and SubjectKeyId should match for a self-signed CA
+	authorityKeyId := publicKeyHash
+	subjectKeyId := publicKeyHash
+	rootcaTemplate := newSigningCertificateTemplateForDuration(subject, caLifetime, time.Now, authorityKeyId, subjectKeyId)
+	rootcaCert, err := signCertificate(rootcaTemplate, rootcaPublicKey, rootcaTemplate, rootcaPrivateKey)
+	if err != nil {
+		return nil, err
+	}
+	caConfig := &TLSCertificateConfig{
+		Certs: []*x509.Certificate{rootcaCert},
+		Key:   rootcaPrivateKey,
+	}
+	return caConfig, nil
+}
+
+func MakeCAConfigForDuration(name string, caLifetime time.Duration, issuer *CA) (*TLSCertificateConfig, error) {
+	// Create CA cert
+	signerPublicKey, signerPrivateKey, publicKeyHash, err := newKeyPairWithHash()
+	if err != nil {
+		return nil, err
+	}
+	authorityKeyId := issuer.Config.Certs[0].SubjectKeyId
+	subjectKeyId := publicKeyHash
+	signerTemplate := newSigningCertificateTemplateForDuration(pkix.Name{CommonName: name}, caLifetime, time.Now, authorityKeyId, subjectKeyId)
+	signerCert, err := issuer.signCertificate(signerTemplate, signerPublicKey)
+	if err != nil {
+		return nil, err
+	}
+	signerConfig := &TLSCertificateConfig{
+		Certs: append([]*x509.Certificate{signerCert}, issuer.Config.Certs...),
+		Key:   signerPrivateKey,
+	}
+	return signerConfig, nil
+}
+
+func (ca *CA) EnsureServerCert(certFile, keyFile string, hostnames sets.String, expireDays int) (*TLSCertificateConfig, bool, error) {
+	certConfig, err := GetServerCert(certFile, keyFile, hostnames)
+	if err != nil {
+		certConfig, err = ca.MakeAndWriteServerCert(certFile, keyFile, hostnames, expireDays)
+		return certConfig, true, err
+	}
+
+	return certConfig, false, nil
+}
+
+func GetServerCert(certFile, keyFile string, hostnames sets.String) (*TLSCertificateConfig, error) {
+	server, err := GetTLSCertificateConfig(certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	cert := server.Certs[0]
+	ips, dns := IPAddressesDNSNames(hostnames.List())
+	missingIps := ipsNotInSlice(ips, cert.IPAddresses)
+	missingDns := stringsNotInSlice(dns, cert.DNSNames)
+	if len(missingIps) == 0 && len(missingDns) == 0 {
+		klog.V(4).Infof("Found existing server certificate in %s", certFile)
+		return server, nil
+	}
+
+	return nil, fmt.Errorf("Existing server certificate in %s was missing some hostnames (%v) or IP addresses (%v).", certFile, missingDns, missingIps)
+}
+
+func (ca *CA) MakeAndWriteServerCert(certFile, keyFile string, hostnames sets.String, expireDays int) (*TLSCertificateConfig, error) {
+	klog.V(4).Infof("Generating server certificate in %s, key in %s", certFile, keyFile)
+
+	server, err := ca.MakeServerCert(hostnames, expireDays)
+	if err != nil {
+		return nil, err
+	}
+	if err := server.WriteCertConfigFile(certFile, keyFile); err != nil {
+		return server, err
+	}
+	return server, nil
+}
+
+// CertificateExtensionFunc is passed a certificate that it may extend, or return an error
+// if the extension attempt failed.
+type CertificateExtensionFunc func(*x509.Certificate) error
+
+func (ca *CA) MakeServerCert(hostnames sets.String, expireDays int, fns ...CertificateExtensionFunc) (*TLSCertificateConfig, error) {
+	serverPublicKey, serverPrivateKey, publicKeyHash, _ := newKeyPairWithHash()
+	authorityKeyId := ca.Config.Certs[0].SubjectKeyId
+	subjectKeyId := publicKeyHash
+	serverTemplate := newServerCertificateTemplate(pkix.Name{CommonName: hostnames.List()[0]}, hostnames.List(), expireDays, time.Now, authorityKeyId, subjectKeyId)
+	for _, fn := range fns {
+		if err := fn(serverTemplate); err != nil {
+			return nil, err
+		}
+	}
+	serverCrt, err := ca.signCertificate(serverTemplate, serverPublicKey)
+	if err != nil {
+		return nil, err
+	}
+	server := &TLSCertificateConfig{
+		Certs: append([]*x509.Certificate{serverCrt}, ca.Config.Certs...),
+		Key:   serverPrivateKey,
+	}
+	return server, nil
+}
+
+func (ca *CA) MakeServerCertForDuration(hostnames sets.String, lifetime time.Duration, fns ...CertificateExtensionFunc) (*TLSCertificateConfig, error) {
+	serverPublicKey, serverPrivateKey, publicKeyHash, _ := newKeyPairWithHash()
+	authorityKeyId := ca.Config.Certs[0].SubjectKeyId
+	subjectKeyId := publicKeyHash
+	serverTemplate := newServerCertificateTemplateForDuration(pkix.Name{CommonName: hostnames.List()[0]}, hostnames.List(), lifetime, time.Now, authorityKeyId, subjectKeyId)
+	for _, fn := range fns {
+		if err := fn(serverTemplate); err != nil {
+			return nil, err
+		}
+	}
+	serverCrt, err := ca.signCertificate(serverTemplate, serverPublicKey)
+	if err != nil {
+		return nil, err
+	}
+	server := &TLSCertificateConfig{
+		Certs: append([]*x509.Certificate{serverCrt}, ca.Config.Certs...),
+		Key:   serverPrivateKey,
+	}
+	return server, nil
+}
+
+func (ca *CA) EnsureClientCertificate(certFile, keyFile string, u user.Info, expireDays int) (*TLSCertificateConfig, bool, error) {
+	certConfig, err := GetTLSCertificateConfig(certFile, keyFile)
+	if err != nil {
+		certConfig, err = ca.MakeClientCertificate(certFile, keyFile, u, expireDays)
+		return certConfig, true, err // true indicates we wrote the files.
+	}
+
+	return certConfig, false, nil
+}
+
+func (ca *CA) MakeClientCertificate(certFile, keyFile string, u user.Info, expireDays int) (*TLSCertificateConfig, error) {
+	klog.V(4).Infof("Generating client cert in %s and key in %s", certFile, keyFile)
+	// ensure parent dirs
+	if err := os.MkdirAll(filepath.Dir(certFile), os.FileMode(0755)); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(filepath.Dir(keyFile), os.FileMode(0755)); err != nil {
+		return nil, err
+	}
+
+	clientPublicKey, clientPrivateKey, _ := NewKeyPair()
+	clientTemplate := newClientCertificateTemplate(userToSubject(u), expireDays, time.Now)
+	clientCrt, err := ca.signCertificate(clientTemplate, clientPublicKey)
+	if err != nil {
+		return nil, err
+	}
+
+	certData, err := EncodeCertificates(clientCrt)
+	if err != nil {
+		return nil, err
+	}
+	keyData, err := encodeKey(clientPrivateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = ioutil.WriteFile(certFile, certData, os.FileMode(0644)); err != nil {
+		return nil, err
+	}
+	if err = ioutil.WriteFile(keyFile, keyData, os.FileMode(0600)); err != nil {
+		return nil, err
+	}
+
+	return GetTLSCertificateConfig(certFile, keyFile)
+}
+
+func (ca *CA) MakeClientCertificateForDuration(u user.Info, lifetime time.Duration) (*TLSCertificateConfig, error) {
+	clientPublicKey, clientPrivateKey, _ := NewKeyPair()
+	clientTemplate := newClientCertificateTemplateForDuration(userToSubject(u), lifetime, time.Now)
+	clientCrt, err := ca.signCertificate(clientTemplate, clientPublicKey)
+	if err != nil {
+		return nil, err
+	}
+
+	certData, err := EncodeCertificates(clientCrt)
+	if err != nil {
+		return nil, err
+	}
+	keyData, err := encodeKey(clientPrivateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return GetTLSCertificateConfigFromBytes(certData, keyData)
+}
+
+type sortedForDER []string
+
+func (s sortedForDER) Len() int {
+	return len(s)
+}
+func (s sortedForDER) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+func (s sortedForDER) Less(i, j int) bool {
+	l1 := len(s[i])
+	l2 := len(s[j])
+	if l1 == l2 {
+		return s[i] < s[j]
+	}
+	return l1 < l2
+}
+
+func userToSubject(u user.Info) pkix.Name {
+	// Ok we are going to order groups in a peculiar way here to workaround a
+	// 2 bugs, 1 in golang (https://github.com/golang/go/issues/24254) which
+	// incorrectly encodes Multivalued RDNs and another in GNUTLS clients
+	// which are too picky (https://gitlab.com/gnutls/gnutls/issues/403)
+	// and try to "correct" this issue when reading client certs.
+	//
+	// This workaround should be killed once Golang's pkix module is fixed to
+	// generate a correct DER encoding.
+	//
+	// The workaround relies on the fact that the first octect that differs
+	// between the encoding of two group RDNs will end up being the encoded
+	// length which is directly related to the group name's length. So we'll
+	// sort such that shortest names come first.
+	ugroups := u.GetGroups()
+	groups := make([]string, len(ugroups))
+	copy(groups, ugroups)
+	sort.Sort(sortedForDER(groups))
+
+	return pkix.Name{
+		CommonName:   u.GetName(),
+		SerialNumber: u.GetUID(),
+		Organization: groups,
+	}
+}
+
+func (ca *CA) signCertificate(template *x509.Certificate, requestKey crypto.PublicKey) (*x509.Certificate, error) {
+	// Increment and persist serial
+	serial, err := ca.SerialGenerator.Next(template)
+	if err != nil {
+		return nil, err
+	}
+	template.SerialNumber = big.NewInt(serial)
+	return signCertificate(template, requestKey, ca.Config.Certs[0], ca.Config.Key)
+}
+
+func NewKeyPair() (crypto.PublicKey, crypto.PrivateKey, error) {
+	return newRSAKeyPair()
+}
+
+func newKeyPairWithHash() (crypto.PublicKey, crypto.PrivateKey, []byte, error) {
+	publicKey, privateKey, err := newRSAKeyPair()
+	var publicKeyHash []byte
+	if err == nil {
+		hash := sha1.New()
+		hash.Write(publicKey.N.Bytes())
+		publicKeyHash = hash.Sum(nil)
+	}
+	return publicKey, privateKey, publicKeyHash, err
+}
+
+func newRSAKeyPair() (*rsa.PublicKey, *rsa.PrivateKey, error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, keyBits)
+	if err != nil {
+		return nil, nil, err
+	}
+	return &privateKey.PublicKey, privateKey, nil
+}
+
+// Can be used for CA or intermediate signing certs
+func newSigningCertificateTemplateForDuration(subject pkix.Name, caLifetime time.Duration, currentTime func() time.Time, authorityKeyId, subjectKeyId []byte) *x509.Certificate {
+	return &x509.Certificate{
+		Subject: subject,
+
+		SignatureAlgorithm: x509.SHA256WithRSA,
+
+		NotBefore: currentTime().Add(-1 * time.Second),
+		NotAfter:  currentTime().Add(caLifetime),
+
+		// Specify a random serial number to avoid the same issuer+serial
+		// number referring to different certs in a chain of trust if the
+		// signing certificate is ever rotated.
+		SerialNumber: big.NewInt(randomSerialNumber()),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+
+		AuthorityKeyId: authorityKeyId,
+		SubjectKeyId:   subjectKeyId,
+	}
+}
+
+// Can be used for ListenAndServeTLS
+func newServerCertificateTemplate(subject pkix.Name, hosts []string, expireDays int, currentTime func() time.Time, authorityKeyId, subjectKeyId []byte) *x509.Certificate {
+	var lifetimeInDays = DefaultCertificateLifetimeInDays
+	if expireDays > 0 {
+		lifetimeInDays = expireDays
+	}
+
+	if lifetimeInDays > DefaultCertificateLifetimeInDays {
+		warnAboutCertificateLifeTime(subject.CommonName, DefaultCertificateLifetimeInDays)
+	}
+
+	lifetime := time.Duration(lifetimeInDays) * 24 * time.Hour
+
+	return newServerCertificateTemplateForDuration(subject, hosts, lifetime, currentTime, authorityKeyId, subjectKeyId)
+}
+
+// Can be used for ListenAndServeTLS
+func newServerCertificateTemplateForDuration(subject pkix.Name, hosts []string, lifetime time.Duration, currentTime func() time.Time, authorityKeyId, subjectKeyId []byte) *x509.Certificate {
+	template := &x509.Certificate{
+		Subject: subject,
+
+		SignatureAlgorithm: x509.SHA256WithRSA,
+
+		NotBefore:    currentTime().Add(-1 * time.Second),
+		NotAfter:     currentTime().Add(lifetime),
+		SerialNumber: big.NewInt(1),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+
+		AuthorityKeyId: authorityKeyId,
+		SubjectKeyId:   subjectKeyId,
+	}
+
+	template.IPAddresses, template.DNSNames = IPAddressesDNSNames(hosts)
+
+	return template
+}
+
+func IPAddressesDNSNames(hosts []string) ([]net.IP, []string) {
+	ips := []net.IP{}
+	dns := []string{}
+	for _, host := range hosts {
+		if ip := net.ParseIP(host); ip != nil {
+			ips = append(ips, ip)
+		} else {
+			dns = append(dns, host)
+		}
+	}
+
+	// Include IP addresses as DNS subjectAltNames in the cert as well, for the sake of Python, Windows (< 10), and unnamed other libraries
+	// Ensure these technically invalid DNS subjectAltNames occur after the valid ones, to avoid triggering cert errors in Firefox
+	// See https://bugzilla.mozilla.org/show_bug.cgi?id=1148766
+	for _, ip := range ips {
+		dns = append(dns, ip.String())
+	}
+
+	return ips, dns
+}
+
+func CertsFromPEM(pemCerts []byte) ([]*x509.Certificate, error) {
+	ok := false
+	certs := []*x509.Certificate{}
+	for len(pemCerts) > 0 {
+		var block *pem.Block
+		block, pemCerts = pem.Decode(pemCerts)
+		if block == nil {
+			break
+		}
+		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
+			continue
+		}
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return certs, err
+		}
+
+		certs = append(certs, cert)
+		ok = true
+	}
+
+	if !ok {
+		return certs, errors.New("Could not read any certificates")
+	}
+	return certs, nil
+}
+
+// Can be used as a certificate in http.Transport TLSClientConfig
+func newClientCertificateTemplate(subject pkix.Name, expireDays int, currentTime func() time.Time) *x509.Certificate {
+	var lifetimeInDays = DefaultCertificateLifetimeInDays
+	if expireDays > 0 {
+		lifetimeInDays = expireDays
+	}
+
+	if lifetimeInDays > DefaultCertificateLifetimeInDays {
+		warnAboutCertificateLifeTime(subject.CommonName, DefaultCertificateLifetimeInDays)
+	}
+
+	lifetime := time.Duration(lifetimeInDays) * 24 * time.Hour
+
+	return newClientCertificateTemplateForDuration(subject, lifetime, currentTime)
+}
+
+// Can be used as a certificate in http.Transport TLSClientConfig
+func newClientCertificateTemplateForDuration(subject pkix.Name, lifetime time.Duration, currentTime func() time.Time) *x509.Certificate {
+	return &x509.Certificate{
+		Subject: subject,
+
+		SignatureAlgorithm: x509.SHA256WithRSA,
+
+		NotBefore:    currentTime().Add(-1 * time.Second),
+		NotAfter:     currentTime().Add(lifetime),
+		SerialNumber: big.NewInt(1),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+	}
+}
+
+func warnAboutCertificateLifeTime(name string, defaultLifetimeInDays int) {
+	defaultLifetimeInYears := defaultLifetimeInDays / 365
+	fmt.Fprintf(os.Stderr, "WARNING: Validity period of the certificate for %q is greater than %d years!\n", name, defaultLifetimeInYears)
+	fmt.Fprintln(os.Stderr, "WARNING: By security reasons it is strongly recommended to change this period and make it smaller!")
+}
+
+func signCertificate(template *x509.Certificate, requestKey crypto.PublicKey, issuer *x509.Certificate, issuerKey crypto.PrivateKey) (*x509.Certificate, error) {
+	derBytes, err := x509.CreateCertificate(rand.Reader, template, issuer, requestKey, issuerKey)
+	if err != nil {
+		return nil, err
+	}
+	certs, err := x509.ParseCertificates(derBytes)
+	if err != nil {
+		return nil, err
+	}
+	if len(certs) != 1 {
+		return nil, errors.New("Expected a single certificate")
+	}
+	return certs[0], nil
+}
+
+func EncodeCertificates(certs ...*x509.Certificate) ([]byte, error) {
+	b := bytes.Buffer{}
+	for _, cert := range certs {
+		if err := pem.Encode(&b, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil {
+			return []byte{}, err
+		}
+	}
+	return b.Bytes(), nil
+}
+func encodeKey(key crypto.PrivateKey) ([]byte, error) {
+	b := bytes.Buffer{}
+	switch key := key.(type) {
+	case *ecdsa.PrivateKey:
+		keyBytes, err := x509.MarshalECPrivateKey(key)
+		if err != nil {
+			return []byte{}, err
+		}
+		if err := pem.Encode(&b, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes}); err != nil {
+			return b.Bytes(), err
+		}
+	case *rsa.PrivateKey:
+		if err := pem.Encode(&b, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}); err != nil {
+			return []byte{}, err
+		}
+	default:
+		return []byte{}, errors.New("Unrecognized key type")
+
+	}
+	return b.Bytes(), nil
+}
+
+func writeCertificates(f io.Writer, certs ...*x509.Certificate) error {
+	bytes, err := EncodeCertificates(certs...)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(bytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+func writeKeyFile(f io.Writer, key crypto.PrivateKey) error {
+	bytes, err := encodeKey(key)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(bytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func stringsNotInSlice(needles []string, haystack []string) []string {
+	missing := []string{}
+	for _, needle := range needles {
+		if !stringInSlice(needle, haystack) {
+			missing = append(missing, needle)
+		}
+	}
+	return missing
+}
+
+func stringInSlice(needle string, haystack []string) bool {
+	for _, straw := range haystack {
+		if needle == straw {
+			return true
+		}
+	}
+	return false
+}
+
+func ipsNotInSlice(needles []net.IP, haystack []net.IP) []net.IP {
+	missing := []net.IP{}
+	for _, needle := range needles {
+		if !ipInSlice(needle, haystack) {
+			missing = append(missing, needle)
+		}
+	}
+	return missing
+}
+
+func ipInSlice(needle net.IP, haystack []net.IP) bool {
+	for _, straw := range haystack {
+		if needle.Equal(straw) {
+			return true
+		}
+	}
+	return false
+}
diff --git a/vendor/github.com/openshift/library-go/pkg/crypto/rotation.go b/vendor/github.com/openshift/library-go/pkg/crypto/rotation.go
new file mode 100644
index 000000000..0aa127037
--- /dev/null
+++ b/vendor/github.com/openshift/library-go/pkg/crypto/rotation.go
@@ -0,0 +1,20 @@
+package crypto
+
+import (
+	"crypto/x509"
+	"time"
+)
+
+// FilterExpiredCerts checks are all certificates in the bundle valid, i.e. they have not expired.
+// The function returns new bundle with only valid certificates or error if no valid certificate is found.
+func FilterExpiredCerts(certs ...*x509.Certificate) []*x509.Certificate {
+	currentTime := time.Now()
+	var validCerts []*x509.Certificate
+	for _, c := range certs {
+		if c.NotAfter.After(currentTime) {
+			validCerts = append(validCerts, c)
+		}
+	}
+
+	return validCerts
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index b7e733c97..5224c693e 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -281,6 +281,7 @@ github.com/openshift/hive/apis/scheme
 # github.com/openshift/library-go v0.0.0-20210330121802-ebbc677c82a5 => github.com/openshift/library-go v0.0.0-20200918101923-1e4c94603efe
 ## explicit
 github.com/openshift/library-go/pkg/authorization/authorizationutil
+github.com/openshift/library-go/pkg/crypto
 # github.com/pkg/errors v0.9.1
 github.com/pkg/errors
 # github.com/pmezard/go-difflib v1.0.0