Merge version v0.8.0

aenix-io · Nov 9, 2020 · 8ab2b8e · 8ab2b8e
2 parents c181314 + f974696
commit 8ab2b8e
Show file tree

Hide file tree

Showing 18 changed files with 480 additions and 210 deletions.
diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ Deploy Kubernetes in Kubernetes using Helm
 
 ```bash
 helm repo add kvaps https://kvaps.github.io/charts
-helm install foo kvaps/kubernetes --version 0.7.0 \
+helm install foo kvaps/kubernetes --version 0.8.0 \
   --namespace foo \
   --create-namespace \
   --set persistence.storageClassName=local-path

diff --git a/deploy/helm/kubernetes/Chart.yaml b/deploy/helm/kubernetes/Chart.yaml
@@ -1,6 +1,6 @@
 name: kubernetes
 description: Production-Grade Container Scheduling and Management
-version: 0.7.0
+version: 0.8.0
 appVersion: 1.19.3
 icon: https://upload.wikimedia.org/wikipedia/commons/thumb/3/39/Kubernetes_logo_without_workmark.svg/723px-Kubernetes_logo_without_workmark.svg.png
 keywords:

diff --git a/deploy/helm/kubernetes/scripts/configure-cluster.sh b/deploy/helm/kubernetes/scripts/configure-cluster.sh
@@ -1,80 +1,15 @@
 #!/bin/sh
 set -e
 set -x
-
-# ------------------------------------------------------------------------------
-# Setup environment
-# ------------------------------------------------------------------------------
-
-mkdir -p /etc/kubernetes/pki
-ln -sf /pki/apiserver-etcd-client/tls.crt /etc/kubernetes/pki/apiserver-etcd-client.crt
-ln -sf /pki/apiserver-etcd-client/tls.key /etc/kubernetes/pki/apiserver-etcd-client.key
-ln -sf /pki/apiserver-kubelet-client/tls.crt /etc/kubernetes/pki/apiserver-kubelet-client.crt
-ln -sf /pki/apiserver-kubelet-client/tls.key /etc/kubernetes/pki/apiserver-kubelet-client.key
-ln -sf /pki/apiserver/tls.crt /etc/kubernetes/pki/apiserver.crt
-ln -sf /pki/apiserver/tls.key /etc/kubernetes/pki/apiserver.key
-ln -sf /pki/ca/tls.crt /etc/kubernetes/pki/ca.crt
-ln -sf /pki/ca/tls.key /etc/kubernetes/pki/ca.key
-ln -sf /pki/front-proxy-ca/tls.key /etc/kubernetes/pki/front-proxy-ca.crt
-ln -sf /pki/front-proxy-ca/tls.crt /etc/kubernetes/pki/front-proxy-ca.key
-ln -sf /pki/front-proxy-client/tls.key /etc/kubernetes/pki/front-proxy-client.crt
-ln -sf /pki/front-proxy-client/tls.crt /etc/kubernetes/pki/front-proxy-client.key
+ENDPOINT=$(awk -F'[ "]+' '$1 == "controlPlaneEndpoint:" {print $2}' /config/kubeadmcfg.yaml)
 
 # ------------------------------------------------------------------------------
 # Update secrets and component configs
 # ------------------------------------------------------------------------------
 
-cat >kubeadmcfg.yaml <<EOT
-apiVersion: "kubeadm.k8s.io/v1beta2" 
-kind: ClusterConfiguration 
-imageRepository: k8s.gcr.io 
-controlPlaneEndpoint: "${FULL_NAME}-apiserver:6443"
-EOT
-
-{{- if .Values.apiServer.enabled }}{{"\n"}}
-# generate sa key
-if ! kubectl get secret "${FULL_NAME}-pki-sa" >/dev/null; then
-  kubeadm init phase certs sa
-  kubectl create secret generic "${FULL_NAME}-pki-sa" --from-file=/etc/kubernetes/pki/sa.pub --from-file=/etc/kubernetes/pki/sa.key
-fi
-{{- end }}
-
-# generate cluster-admin kubeconfig
-rm -f /etc/kubernetes/admin.conf
-kubeadm init phase kubeconfig admin --config kubeadmcfg.yaml
-kubectl --kubeconfig=/etc/kubernetes/admin.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443"
-kubectl create secret generic "${FULL_NAME}-admin-conf" --from-file=/etc/kubernetes/admin.conf --dry-run=client -o yaml | kubectl apply -f -
-
-{{- if .Values.controllerManager.enabled }}{{"\n"}}
-# generate controller-manager kubeconfig
-rm -f /etc/kubernetes/controller-manager.conf
-kubeadm init phase kubeconfig controller-manager --config kubeadmcfg.yaml
-kubectl --kubeconfig=/etc/kubernetes/controller-manager.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443"
-kubectl create secret generic "${FULL_NAME}-controller-manager-conf" --from-file=/etc/kubernetes/controller-manager.conf --dry-run=client -o yaml | kubectl apply -f -
-{{- end }}
-
-{{- if .Values.scheduler.enabled }}{{"\n"}}
-# generate scheduler kubeconfig
-rm -f /etc/kubernetes/scheduler.conf
-kubeadm init phase kubeconfig scheduler --config kubeadmcfg.yaml
-kubectl --kubeconfig=/etc/kubernetes/scheduler.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443"
-kubectl create secret generic "${FULL_NAME}-scheduler-conf" --from-file=/etc/kubernetes/scheduler.conf --dry-run=client -o yaml | kubectl apply -f -
-{{- end }}
-
-{{- if .Values.konnectivityServer.enabled }}{{"\n"}}
-# generate konnectivity-server kubeconfig
-openssl req -subj "/CN=system:konnectivity-server" -new -newkey rsa:2048 -nodes -out konnectivity.csr -keyout konnectivity.key -out konnectivity.csr
-openssl x509 -req -in konnectivity.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -out konnectivity.crt -days 375 -sha256
-kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config set-credentials system:konnectivity-server --client-certificate konnectivity.crt --client-key konnectivity.key --embed-certs=true
-kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443" --certificate-authority /etc/kubernetes/pki/ca.crt --embed-certs=true
-kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config set-context system:konnectivity-server@kubernetes --cluster kubernetes --user system:konnectivity-server
-kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config use-context system:konnectivity-server@kubernetes
-kubectl create secret generic "${FULL_NAME}-konnectivity-server-conf" --from-file=/etc/kubernetes/konnectivity-server.conf --dry-run=client -o yaml | kubectl apply -f -
-{{- end }}
-
 # wait for cluster
-echo "Waiting for api-server endpoint ${FULL_NAME}-apiserver:6443..."
-until kubectl --kubeconfig /etc/kubernetes/admin.conf cluster-info >/dev/null 2>/dev/null; do
+echo "Waiting for api-server endpoint ${ENDPOINT}..."
+until kubectl cluster-info >/dev/null 2>/dev/null; do
   sleep 1
 done
 
@@ -84,53 +19,72 @@ done
 export KUBECONFIG=/etc/kubernetes/admin.conf
 
 # upload configuration
+# TODO: https://github.com/kvaps/kubernetes-in-kubernetes/issues/6
 kubeadm init phase upload-config kubeadm --config /config/kubeadmcfg.yaml
-kubectl --kubeconfig /etc/kubernetes/admin.conf patch configmap -n kube-system kubeadm-config \
+kubectl patch configmap -n kube-system kubeadm-config \
   -p '{"data":{"ClusterStatus":"apiEndpoints: {}\napiVersion: kubeadm.k8s.io/v1beta2\nkind: ClusterStatus"}}'
 
 # upload configuration
+# TODO: https://github.com/kvaps/kubernetes-in-kubernetes/issues/5
 kubeadm init phase upload-config kubelet --config /config/kubeadmcfg.yaml -v1 2>&1 |
   while read line; do echo "$line" | grep 'Preserving the CRISocket information for the control-plane node' && killall kubeadm || echo "$line"; done
 
 # setup bootstrap-tokens
+# TODO: https://github.com/kvaps/kubernetes-in-kubernetes/issues/7
 kubeadm init phase bootstrap-token --config /config/kubeadmcfg.yaml --skip-token-print
 
 # correct apiserver address for the external clients
-tmp="$(mktemp -d)"
-kubectl --kubeconfig "$tmp/kubeconfig" config set clusters..server "https://${CONTROL_PLANE_ENDPOINT:-${FULL_NAME}-apiserver:6443}"
-kubectl --kubeconfig "$tmp/kubeconfig" config set clusters..certificate-authority-data "$(base64 /etc/kubernetes/pki/ca.crt | tr -d '\n')"
-kubectl create configmap cluster-info --from-file="$tmp/kubeconfig" --dry-run=client -o yaml | kubectl --kubeconfig /etc/kubernetes/admin.conf apply -n kube-public -f -
-rm -rf "$tmp"
+kubectl apply -n kube-public -f - <<EOT
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-info
+data:
+  kubeconfig: |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority-data: $(base64 /pki/admin-client/ca.crt | tr -d '\n')
+        server: https://${ENDPOINT}
+      name: ""
+    contexts: null
+    current-context: ""
+    kind: Config
+    preferences: {}
+    users: null
+EOT
 
 {{- if .Values.konnectivityServer.enabled }}{{"\n"}}
 # install konnectivity server
-kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /manifests/konnectivity-server-rbac.yaml
+kubectl apply -f /manifests/konnectivity-server-rbac.yaml
 {{- else }}{{"\n"}}
-kubectl --kubeconfig /etc/kubernetes/admin.conf delete clusterrolebinding/system:konnectivity-server 2>/dev/null || true
+kubectl delete clusterrolebinding/system:konnectivity-server 2>/dev/null || true
 {{- end }}
 
 {{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
 # install konnectivity agent
-kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml
+kubectl apply -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml
 {{- else }}{{"\n"}}
 # uninstall konnectivity agent
-kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-system delete deployment/konnectivity-agent serviceaccount/konnectivity-agent 2>/dev/null || true
+kubectl -n kube-system delete deployment/konnectivity-agent serviceaccount/konnectivity-agent 2>/dev/null || true
 {{- end }}
 
 {{- if .Values.coredns.enabled }}{{"\n"}}
 # install coredns addon
+# TODO: https://github.com/kvaps/kubernetes-in-kubernetes/issues/3
 kubeadm init phase addon coredns --config /config/kubeadmcfg.yaml
 {{- else }}{{"\n"}}
 # uninstall coredns addon
-kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-system delete configmap/coredns deployment/coredns 2>/dev/null || true
+kubectl -n kube-system delete configmap/coredns deployment/coredns 2>/dev/null || true
 {{- end }}
 
 {{- if .Values.kubeProxy.enabled }}{{"\n"}}
 # install kube-proxy addon
+# TODO: https://github.com/kvaps/kubernetes-in-kubernetes/issues/4
 kubeadm init phase addon kube-proxy --config /config/kubeadmcfg.yaml
 {{- else }}{{"\n"}}
 # uninstall kube-proxy addon
-kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-system delete configmap/kube-proxy daemonset/kube-proxy 2>/dev/null || true
+kubectl -n kube-system delete configmap/kube-proxy daemonset/kube-proxy 2>/dev/null || true
 {{- end }}
 
 {{- with .Values.extraManifests }}{{"\n"}}

diff --git a/deploy/helm/kubernetes/templates/admin-configmap.yaml b/deploy/helm/kubernetes/templates/admin-configmap.yaml
@@ -0,0 +1,30 @@
+{{- if .Values.admin.enabled }}
+{{- $fullName := include "kubernetes.fullname" . -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $fullName }}-admin-conf
+data:
+  admin.conf: |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority: /pki/admin-client/ca.crt
+        server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }}
+      name: default-cluster
+    contexts:
+    - context:
+        cluster: default-cluster
+        namespace: default
+        user: default-auth
+      name: default-context
+    current-context: default-context
+    kind: Config
+    preferences: {}
+    users:
+    - name: default-auth
+      user:
+        client-certificate: /pki/admin-client/tls.crt
+        client-key: /pki/admin-client/tls.key
+{{- end }}
diff --git a/deploy/helm/kubernetes/templates/admin-deployment.yaml b/deploy/helm/kubernetes/templates/admin-deployment.yaml
@@ -56,7 +56,7 @@ spec:
         {{- end }}
         imagePullPolicy: {{ .Values.admin.image.PullPolicy }}
         name: admin
-        livenessProbe:
+        readinessProbe:
           exec:
             command:
             - kubectl
@@ -71,25 +71,56 @@ spec:
         env:
         - name: KUBECONFIG
           value: "/etc/kubernetes/admin.conf"
-        - name: FULL_NAME
-          value: "{{ $fullName }}"
         {{- with .Values.admin.extraEnv }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
         volumeMounts:
         - mountPath: /etc/kubernetes/
           name: kubeconfig
           readOnly: true
+        - mountPath: /pki/admin-client
+          name: pki-admin-client
+        - mountPath: /scripts
+          name: scripts
+        {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }}
+        - mountPath: /manifests
+          name: manifests
+        {{- end }}
+        - mountPath: /config
+          name: config
         {{- with .Values.admin.extraVolumeMounts }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
       {{- with .Values.admin.sidecars }}
       {{- toYaml . | nindent 6 }}
       {{- end }}
       volumes:
-      - secret:
-          secretName: "{{ $fullName }}-admin-conf"
+      - configMap:
+          name: "{{ $fullName }}-admin-conf"
         name: kubeconfig
+      - secret:
+          secretName: "{{ $fullName }}-pki-admin-client"
+        name: pki-admin-client
+      - name: scripts
+        configMap:
+          name: "{{ $fullName }}-kubeadm-scripts"
+          defaultMode: 0777
+      {{- if or .Values.extraManifests .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }}
+      - name: manifests
+        projected:
+          sources:
+          {{- if or .Values.extraManifests }}
+          - secret:
+              name: "{{ $fullName }}-extra-manifests"
+          {{- end }}
+          {{- if or .Values.konnectivityServer.enabled .Values.konnectivityAgent.enabled }}
+          - configMap:
+              name: "{{ $fullName }}-konnectivity-manifests"
+          {{- end }}
+      {{- end }}
+      - name: config
+        configMap:
+          name: "{{ $fullName }}-kubeadm-config"
       {{- with .Values.admin.extraVolumes }}
       {{- toYaml . | nindent 6 }}
       {{- end }}

diff --git a/deploy/helm/kubernetes/templates/apiserver-deployment.yaml b/deploy/helm/kubernetes/templates/apiserver-deployment.yaml
@@ -56,7 +56,7 @@ spec:
         - --allow-privileged=true
         - --authorization-mode=Node,RBAC
         - --bind-address=0.0.0.0
-        - --client-ca-file=/pki/apiserver/ca.crt
+        - --client-ca-file=/pki/apiserver-server/ca.crt
         - --enable-admission-plugins=NodeRestriction
         - --enable-bootstrap-token-auth=true
         - --etcd-cafile=/pki/apiserver-etcd-client/ca.crt
@@ -75,14 +75,14 @@ spec:
         - --requestheader-group-headers=X-Remote-Group
         - --requestheader-username-headers=X-Remote-User
         - --secure-port={{ .Values.apiServer.port }}
-        - --service-account-key-file=/pki/sa/sa.pub
+        - --service-account-key-file=/pki/sa/tls.crt
         - --service-cluster-ip-range={{ .Values.apiServer.serviceClusterIPRange }}
-        - --tls-cert-file=/pki/apiserver/tls.crt
-        - --tls-private-key-file=/pki/apiserver/tls.key
+        - --tls-cert-file=/pki/apiserver-server/tls.crt
+        - --tls-private-key-file=/pki/apiserver-server/tls.key
         - --egress-selector-config-file=/etc/kubernetes/egress-selector-configuration.yaml
         {{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
         - --service-account-issuer=api
-        - --service-account-signing-key-file=/pki/sa/sa.key
+        - --service-account-signing-key-file=/pki/sa/tls.key
         - --api-audiences=system:konnectivity-server
         {{- end }}
         {{- if not (hasKey .Values.apiServer.extraArgs "advertise-address") }}
@@ -120,8 +120,8 @@ spec:
           name: apiserver-config
         - mountPath: /pki/front-proxy-client
           name: pki-front-proxy-client
-        - mountPath: /pki/apiserver
-          name: pki-apiserver
+        - mountPath: /pki/apiserver-server
+          name: pki-apiserver-server
         - mountPath: /pki/apiserver-etcd-client
           name: pki-apiserver-etcd-client
         - mountPath: /pki/apiserver-kubelet-client
@@ -146,17 +146,14 @@ spec:
           secretName: "{{ $fullName }}-pki-front-proxy-client"
         name: pki-front-proxy-client
       - secret:
-          secretName: "{{ $fullName }}-pki-apiserver"
-        name: pki-apiserver
+          secretName: "{{ $fullName }}-pki-apiserver-server"
+        name: pki-apiserver-server
       - secret:
           secretName: "{{ $fullName }}-pki-apiserver-etcd-client"
         name: pki-apiserver-etcd-client
       - secret:
           secretName: "{{ $fullName }}-pki-apiserver-kubelet-client"
         name: pki-apiserver-kubelet-client
-      - secret:
-          secretName: "{{ $fullName }}-pki-ca"
-        name: pki-ca
       - secret:
           secretName: "{{ $fullName }}-pki-sa"
         name: pki-sa

diff --git a/deploy/helm/kubernetes/templates/controller-manager-configmap.yaml b/deploy/helm/kubernetes/templates/controller-manager-configmap.yaml
@@ -0,0 +1,30 @@
+{{- if .Values.controllerManager.enabled }}
+{{- $fullName := include "kubernetes.fullname" . -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $fullName }}-controller-manager-conf
+data:
+  controller-manager.conf: |
+    apiVersion: v1
+    clusters:
+    - cluster:
+        certificate-authority: /pki/controller-manager-client/ca.crt
+        server: https://{{ $fullName }}-apiserver:{{ .Values.apiServer.service.port }}
+      name: default-cluster
+    contexts:
+    - context:
+        cluster: default-cluster
+        namespace: default
+        user: default-auth
+      name: default-context
+    current-context: default-context
+    kind: Config
+    preferences: {}
+    users:
+    - name: default-auth
+      user:
+        client-certificate: /pki/controller-manager-client/tls.crt
+        client-key: /pki/controller-manager-client/tls.key
+{{- end }}