Merge pull request stolostron#376 from zhiweiyin318/cert-controller

add log cert controller

Makefile

@@ -93,6 +93,7 @@ build-e2e:
 	go test -c ./test/e2e
 
 test-e2e: build-e2e deploy-hub deploy-klusterlet deploy-foundation-hub deploy-foundation-webhook deploy-foundation-agent
+	deploy/foundation/scripts/install-check.sh
 	./e2e.test -test.v -ginkgo.v
 
 ############################################################

cmd/agent/agent.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"context"
+	"k8s.io/component-base/logs"
 	"os"
 	"time"
 
@@ -68,6 +69,8 @@ func main() {
 	o := options.NewAgentOptions()
 	o.AddFlags(pflag.CommandLine)
 
+	logs.InitLogs()
+	defer logs.FlushLogs()
 	ctx := signals.SetupSignalHandler()
 	startManager(o, ctx)
 }

cmd/controller/app/options/options.go

@@ -10,6 +10,7 @@ import (
 type ControllerRunOptions struct {
 	KubeConfig           string
 	CAFile               string
+	LogCertSecret        string
 	EnableInventory      bool
 	EnableLeaderElection bool
 	EnableRBAC           bool
@@ -22,6 +23,7 @@ func NewControllerRunOptions() *ControllerRunOptions {
 	return &ControllerRunOptions{
 		KubeConfig:           "",
 		CAFile:               "/var/run/agent/ca.crt",
+		LogCertSecret:        "ocm-klusterlet-self-signed-secrets",
 		EnableInventory:      true,
 		EnableLeaderElection: true,
 		EnableRBAC:           false,
@@ -36,6 +38,8 @@ func (o *ControllerRunOptions) AddFlags(fs *pflag.FlagSet) {
 		"The kubeconfig to connect to cluster to watch/apply resources.")
 	fs.StringVar(&o.CAFile, "agent-cafile", o.CAFile, ""+
 		"Agent CA file.")
+	fs.StringVar(&o.LogCertSecret, "log-cert-secret", o.LogCertSecret,
+		"log cert secret name.")
 	fs.BoolVar(&o.EnableInventory, "enable-inventory", o.EnableInventory,
 		"enable multi-cluster inventory")
 	fs.BoolVar(&o.EnableLeaderElection, "enable-leader-election", o.EnableLeaderElection,

cmd/controller/app/server.go

@@ -4,8 +4,7 @@ package app
 
 import (
 	"context"
-	"io/ioutil"
-	"path"
+	"github.com/open-cluster-management/multicloud-operators-foundation/pkg/controllers/certrotation"
 	"time"
 
 	clusterv1client "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
@@ -90,11 +89,6 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 	clusterInformers := clusterv1informers.NewSharedInformerFactory(clusterClient, 10*time.Minute)
 	kubeInfomers := kubeinformers.NewSharedInformerFactory(kubeClient, 10*time.Minute)
 
-	caData, err := GetAgentCA(o.CAFile)
-	if err != nil {
-		klog.Warningf("unable to get foundation agent server CA file: %v", err)
-	}
-
 	mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
 		Scheme:                 scheme,
 		LeaderElectionID:       "foundation-controller",
@@ -139,7 +133,7 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 		}
 	}
 
-	if err = clusterinfo.SetupWithManager(mgr, caData); err != nil {
+	if err = clusterinfo.SetupWithManager(mgr, o.LogCertSecret); err != nil {
 		klog.Errorf("unable to setup clusterInfo reconciler: %v", err)
 		return err
 	}
@@ -177,6 +171,12 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 		klog.Errorf("unable to setup gc reconciler: %v", err)
 		return err
 	}
+
+	if err = certrotation.SetupWithManager(mgr, o.LogCertSecret); err != nil {
+		klog.Errorf("unable to setup cert rotation reconciler: %v", err)
+		return err
+	}
+
 	go func() {
 		<-mgr.Elected()
 		go clusterInformers.Start(ctx.Done())
@@ -196,11 +196,3 @@ func Run(o *options.ControllerRunOptions, ctx context.Context) error {
 
 	return nil
 }
-
-func GetAgentCA(caFile string) ([]byte, error) {
-	pemBlock, err := ioutil.ReadFile(path.Clean(caFile))
-	if err != nil {
-		return nil, err
-	}
-	return pemBlock, nil
-}

deploy/foundation/hub/resources/clusterviewv1-apiservice.yaml

@@ -2,12 +2,13 @@ apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
   name: v1.clusterview.open-cluster-management.io
+  annotations:
+    "service.beta.openshift.io/inject-cabundle": "true"
 spec:
   group: clusterview.open-cluster-management.io
   version: v1
   service:
     namespace: open-cluster-management
     name: foundation-proxyserver
-  insecureSkipTLSVerify: true
   groupPriorityMinimum: 10
   versionPriority: 20

deploy/foundation/hub/resources/clusterviewv1alpha1-apiservice.yaml

@@ -2,12 +2,13 @@ apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
   name: v1alpha1.clusterview.open-cluster-management.io
+  annotations:
+    "service.beta.openshift.io/inject-cabundle": "true"
 spec:
   group: clusterview.open-cluster-management.io
   version: v1alpha1
   service:
     namespace: open-cluster-management
     name: foundation-proxyserver
-  insecureSkipTLSVerify: true
   groupPriorityMinimum: 10
   versionPriority: 20

deploy/foundation/hub/resources/proxyserver-apiservice.yaml

@@ -2,12 +2,13 @@ apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
   name: v1beta1.proxy.open-cluster-management.io
+  annotations:
+    "service.beta.openshift.io/inject-cabundle": "true"
 spec:
   group: proxy.open-cluster-management.io
   version: v1beta1
   service:
     namespace: open-cluster-management
     name: foundation-proxyserver
-  insecureSkipTLSVerify: true
   groupPriorityMinimum: 10000
   versionPriority: 20

deploy/foundation/hub/resources/proxyserver-service.yaml

@@ -3,6 +3,8 @@ apiVersion: v1
 metadata:
   name: foundation-proxyserver
   namespace: open-cluster-management
+  annotations:
+    "service.beta.openshift.io/serving-cert-secret-name": foundation-proxyserver
 spec:
   selector:
     app: foundation-proxyserver

deploy/foundation/hub/resources/proxyserver.yaml

@@ -22,9 +22,17 @@ spec:
         imagePullPolicy: IfNotPresent
         args:
           - "/proxyserver"
-          - "--agent-cert-dir=/tmp/agent-cert"
-          - "--cert-dir=/tmp/cert"
           - "--secure-port=6443"
+          - "--tls-cert-file=/var/run/apiservice/tls.crt"
+          - "--tls-private-key-file=/var/run/apiservice/tls.key"
+          - "--agent-cafile=/var/run/klusterlet/ca.crt"
+          - "--agent-certfile=/var/run/klusterlet/tls.crt"
+          - "--agent-keyfile=/var/run/klusterlet/tls.key"
+        volumeMounts:
+        - mountPath: /var/run/klusterlet
+          name: klusterlet-certs
+        - mountPath: /var/run/apiservice
+          name: apiservice-certs
         livenessProbe:
           httpGet:
             path: /healthz
@@ -38,3 +46,12 @@ spec:
             scheme: HTTPS
             port: 6443
           initialDelaySeconds: 2
+      volumes:
+      - name: klusterlet-certs
+        secret:
+          defaultMode: 420
+          secretName: ocm-klusterlet-self-signed-secrets
+      - name: apiservice-certs
+        secret:
+          defaultMode: 420
+          secretName: foundation-proxyserver

deploy/foundation/klusterlet/resources/agent-manifestwork.yaml

@@ -89,7 +89,6 @@ spec:
                 - "--agent-address=foundation-agent.open-cluster-management-agent.svc"
                 - "--agent-port=443"
                 - "--lease-duration=5"
-                - "--insecure=true"
               volumeMounts:
                 - name: hub-config
                   mountPath: /var/run/hub

deploy/foundation/scripts/install-check.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+
+KUBECTL=${KUBECTL:-kubectl}
+
+
+for i in {1..7}; do
+  echo "############$i  Checking foundation pods"
+  RUNNING_POD=0
+  controller=$($KUBECTL -n open-cluster-management get pods | grep foundation-controller | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+controller))
+  proxyserver=$($KUBECTL -n open-cluster-management get pods | grep foundation-proxyserver | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+proxyserver))
+  webhook=$($KUBECTL -n open-cluster-management get pods | grep foundation-webhook | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+webhook))
+  agent=$($KUBECTL -n open-cluster-management-agent get pods | grep foundation-agent | grep -c "Running")
+  RUNNING_POD=$((RUNNING_POD+agent))
+
+  if [ "${RUNNING_POD}" -eq 4 ]; then
+    break
+  fi
+
+  if [ $i -eq 7 ]; then
+    echo "!!!!!!!!!!  the foundation pods are not ready within 4 minutes"
+    $KUBECTL -n open-cluster-management get pods
+    $KUBECTL -n open-cluster-management get secret
+    $KUBECTL -n open-cluster-management-agent get pods
+    $KUBECTL get mcl
+    $KUBECTL -n cluster1 get manifestworks.work.open-cluster-management.io -o yaml
+    exit 1
+  fi
+  sleep 30
+done
+
+
+echo "############  Foundation is installed successfully!!"
+