From c5dddb61fd05ffd6e3fc60a26873cccb2bdda2fe Mon Sep 17 00:00:00 2001 From: romdalf Date: Sun, 9 Jun 2024 19:51:00 +0200 Subject: [PATCH] 'testing script for vault integration' --- .../vault-encryption-config-bkp.yaml | 8 + .../vault-encryption-config-with_kms.yaml | 13 ++ .../encryption/vault-encryption-config.yaml | 5 - configuration/k8s/kind/kind-vault.yaml | 4 +- configuration/kleidi/vault-config.json | 2 +- configuration/testenv4kvault.sh | 192 ++++++++++++++---- docs/architecture.md | 2 +- docs/security.md | 2 +- docs/vault.md | 4 +- 9 files changed, 176 insertions(+), 56 deletions(-) create mode 100644 configuration/k8s/encryption/vault-encryption-config-bkp.yaml create mode 100644 configuration/k8s/encryption/vault-encryption-config-with_kms.yaml diff --git a/configuration/k8s/encryption/vault-encryption-config-bkp.yaml b/configuration/k8s/encryption/vault-encryption-config-bkp.yaml new file mode 100644 index 0000000..01739ef --- /dev/null +++ b/configuration/k8s/encryption/vault-encryption-config-bkp.yaml @@ -0,0 +1,8 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: + - resources: + - secrets + - configmaps + providers: + - identity: {} diff --git a/configuration/k8s/encryption/vault-encryption-config-with_kms.yaml b/configuration/k8s/encryption/vault-encryption-config-with_kms.yaml new file mode 100644 index 0000000..84d8c9d --- /dev/null +++ b/configuration/k8s/encryption/vault-encryption-config-with_kms.yaml @@ -0,0 +1,13 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: + - resources: + - secrets + - configmaps + providers: + - kms: + apiVersion: v2 + name: kleidi-kms-plugin + endpoint: unix:///tmp/kleidi/kleidi-kms-plugin.socket + timeout: 5s + - identity: {} diff --git a/configuration/k8s/encryption/vault-encryption-config.yaml b/configuration/k8s/encryption/vault-encryption-config.yaml index 84d8c9d..01739ef 100644 --- a/configuration/k8s/encryption/vault-encryption-config.yaml +++ b/configuration/k8s/encryption/vault-encryption-config.yaml @@ -5,9 +5,4 @@ resources: - secrets - configmaps providers: - - kms: - apiVersion: v2 - name: kleidi-kms-plugin - endpoint: unix:///tmp/kleidi/kleidi-kms-plugin.socket - timeout: 5s - identity: {} diff --git a/configuration/k8s/kind/kind-vault.yaml b/configuration/k8s/kind/kind-vault.yaml index 6f25b76..95181f7 100644 --- a/configuration/k8s/kind/kind-vault.yaml +++ b/configuration/k8s/kind/kind-vault.yaml @@ -6,11 +6,11 @@ nodes: image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245 extraMounts: - containerPath: /etc/kubernetes/encryption-config.yaml - hostPath: configuration/k8s/encryption/vault-encryption-config.yaml + hostPath: k8s/encryption/vault-encryption-config.yaml readOnly: true propagation: None - containerPath: /opt/kleidi/config.json - hostPath: configuration/kleidi/vault-config.json + hostPath: kleidi/vault-config.json readOnly: true propagation: None kubeadmConfigPatches: diff --git a/configuration/kleidi/vault-config.json b/configuration/kleidi/vault-config.json index 42ac3e3..5347089 100644 --- a/configuration/kleidi/vault-config.json +++ b/configuration/kleidi/vault-config.json @@ -2,5 +2,5 @@ "Namespace": "", "Transitkey": "kleidi", "Vaultrole": "kleidi", - "Address": "http://192.168.172.243:8200" + "Address": "http://10.89.0.1:8200" } diff --git a/configuration/testenv4kvault.sh b/configuration/testenv4kvault.sh index c456e92..bbd1f7c 100755 --- a/configuration/testenv4kvault.sh +++ b/configuration/testenv4kvault.sh @@ -13,65 +13,169 @@ set -euo pipefail echo echo -e "Test kubernetes environment for kleidi-kms-plugin" +echo echo -e " -> Cleaning any existing vault test env" killall -9 vault ||true + +echo echo -e " -> Cleaning any existing kind test env" -kind delete cluster --quiet --name kleidi-vault +kind delete cluster --name kleidi-vault + +echo +echo -e " -> Cleaning vault-encryption-config.yaml" +cp k8s/encryption/vault-encryption-config-bkp.yaml k8s/encryption/vault-encryption-config.yaml + +echo +echo -e " -> Starting HashiCorp Vault" +nohup vault server -dev -dev-root-token-id=kleidi-demo -dev-listen-address=0.0.0.0:8200 2> /dev/null & +echo -e " -> Sleeping for 10 seconds" +sleep 10 + +echo -e " -> Exporting HashiCorp Vault parameters" +export VAULT_ADDR=http://0.0.0.0:8200 +export VAULT_TOKEN="kleidi-demo" +export VAULT_SKIP_VERITY="true" + +echo +echo -e " -> Enabling HashiCorp Vault transit engine" +vault secrets enable transit + +echo +echo -e " -> Creating kleidi key in HashiCorp Vault transit engine" +vault write -f transit/keys/kleidi + +echo +echo -e " -> Applying kleidi policy in HashiCorp Vault" +vault policy write kleidi vault/vault-policy.hcl + +echo +echo -e " -> Starting k8s instance with Kind" +kind create cluster --config k8s/kind/kind-vault.yaml +echo -e " -> Sleeping for 10 seconds" +sleep 10 -# echo -e " -> Starting HashiCorp Vault" -# nohup vault server -dev -dev-root-token-id=kleidi-demo -dev-listen-address=0.0.0.0:8200 2> /dev/null & -# sleep 3 +echo +echo -ne " -> Checking k8s deployed version (>=1.29)...." +MINORVERSION=`kubectl version -o json |jq -r '.serverVersion.minor'` +NODEVERSION=`kubectl version -o json |jq -r '.serverVersion.gitVersion'` +if [[ ${MINORVERSION} -lt "29" ]] +then + echo -ne "NOK\n" + echo -e " $/!\ kleidi-kms-plugin$ requires kubernetes >=1.29 - current ${NODEVERSION}" + exit +fi +echo -ne ".OK (${NODEVERSION})\n" -# export VAULT_ADDR=http://192.168.172.243:8200 -# export VAULT_TOKEN="kleidi-demo" -# export VAULT_SKIP_VERITY="true" -# echo -e " -> Enabling vault transit engine" -# vault secrets enable transit +echo +echo -e " -> Creating a pre kleidi deployment Secret" +kubectl create secret generic prekleidi -n default --from-literal=mykey=mydata -# echo -e " -> Enabling vault transit engine" -# vault write -f transit/keys/kleidi +echo +echo -e " -> Creating kleidi k8s ServiceAccount/SA Secret/RBAC" +kubectl apply -f k8s/deploy/vault-sa.yaml -# echo -e " -> Applying vault policy" -# vault policy write kleidi configuration/vault/vault-policy.hcl +echo +echo -e " -> Enable k8s auth in HashiCorp Vault" +vault auth enable kubernetes +echo -e " -> Sleeping for 5 seconds" +sleep 5 -# echo -e " -> Starting kind kubernetes instance for vault testing" -# kind create cluster --quiet --config configuration/k8s/kind/kind-vault.yaml -# sleep 3 +echo +echo -e " -> Exporting k8s token, cert, and k8s cluster info HashiCorp Vault k8s auth" +echo -e " -> export kleidi-vault-auth secret token" +export TOKEN=$(kubectl get secret -n kube-system kleidi-vault-auth -o go-template='{{ .data.token }}' | base64 --decode) +echo -e " -> export k8s root CA" +export CERT=$(kubectl get cm kube-root-ca.crt -o jsonpath="{['data']['ca\.crt']}") +echo -e " -> export k8s certificate issuer" +export K8SPORT=$(kubectl config view --raw --minify --flatten --output 'jsonpath={.clusters[].cluster.server}') +export K8SISSU="kubernetes.default.svc.cluster.local" +export K8SHOST=${K8SPORT/127.0.0.1/"$K8SISSU"} +echo -e " -> Sleeping for 5 seconds" +sleep 5 -# echo -ne " -> Checking kubernetes version (>=1.29).." -# MINORVERSION=`kubectl version -o json |jq -r '.serverVersion.minor'` -# NODEVERSION=`kubectl version -o json |jq -r '.serverVersion.gitVersion'` -# if [[ ${MINORVERSION} -lt "29" ]] -# then -# echo -ne "NOK\n" -# echo -e " $/!\ kleidi-kms-plugin$ requires kubernetes >=1.29 - current ${NODEVERSION}" -# exit -# fi -# echo -ne ".OK (${NODEVERSION})\n" +echo +echo -e " -> Write k8s auth config in HashiCorp Vault" +vault write auth/kubernetes/config token_reviewer_jwt="${TOKEN}" kubernetes_host="${K8SHOST}" kubernetes_ca_cert="${CERT}" -# echo -e " -> Creating kubernetes SA/TOKEN/RBAC" -# kubectl apply -f configuration/k8s/deploy/vault-sa.yaml 1> /dev/null +echo +echo -e " -> Create k8s auth role in HashiCorp Vault with kleidi ServiceAccount" +vault write auth/kubernetes/role/kleidi bound_service_account_names=kleidi-vault-auth bound_service_account_namespaces=kube-system policies=kleidi ttl=24h -# echo -e " -> Enable vault kubernetes authentication" -# vault auth enable kubernetes +echo +echo -e " -> Deploy kleidi static pod with HashiCorp Vault integration" +kubectl apply -f k8s/deploy/vault-pod-kleidi-kms.yaml +echo -e " -> Sleeping for 30 seconds to allow pull image" +sleep 30 + +KLEIDI=`kubectl -n kube-system get pod kleidi-kms-plugin --no-headers -ocustom-columns=status:.status.phase` +if [[ ${KLEIDI} == "Running" ]] +then + echo -e " -> kleidi is running" +else + echo -e " -> kleidi is not running" + exit +fi -# echo -e " -> Exporting token, cert, and k8s cluster info" -# export TOKEN=$(kubectl get secret -n kube-system kleidi-vault-auth -o go-template='{{ .data.token }}' | base64 --decode) -# export CERT=$(kubectl get cm kube-root-ca.crt -o jsonpath="{['data']['ca\.crt']}") -# export K8SHOST=$(kubectl config view --raw --minify --flatten --output 'jsonpath={.clusters[].cluster.server}') +echo +echo -e " -> Update vault-encryption-config.yaml with KMS provider" +cp k8s/encryption/vault-encryption-config-with_kms.yaml k8s/encryption/vault-encryption-config.yaml -# echo -e " -> Write vault config for kubernetes authentication" -# vault write auth/kubernetes/config token_reviewer_jwt="${TOKEN}" kubernetes_host="${K8SHOST}" kubernetes_ca_cert="${CERT}" +echo +echo -e " -> Trigger Kind k8s API server restart" +kubectl delete -n kube-system pod/kube-apiserver-kleidi-vault-control-plane +echo -e " -> Sleeping for 10 seconds to allow kube-apiserver to restart" +sleep 10 -# echo -e " -> Link vault config and policy for kubernetes authentication" -# vault write auth/kubernetes/role/kleidi bound_service_account_names=kleidi-vault-auth bound_service_account_namespaces=kube-system policies=kleidi ttl=24h +echo +echo -e " -> Creating a post kleidi deployment Secret" +kubectl create secret generic postkleidi -n default --from-literal=mykey=mydata + +echo +echo -e " -> Checking a pre kleidi deployment Secret" +# kubectl -n kube-system exec etcd-kleidi-vault-control-plane -- sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/etc/kubernetes/pki/etcd/ca.crt' ETCDCTL_CERT='/etc/kubernetes/pki/etcd/server.crt' ETCDCTL_KEY='/etc/kubernetes/pki/etcd/server.key' ETCDCTL_API=3 etcdctl get /registry/secrets/default/prekleidi" | hexdump -C + +if kubectl -n kube-system exec etcd-kleidi-vault-control-plane -- sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/etc/kubernetes/pki/etcd/ca.crt' ETCDCTL_CERT='/etc/kubernetes/pki/etcd/server.crt' ETCDCTL_KEY='/etc/kubernetes/pki/etcd/server.key' ETCDCTL_API=3 etcdctl get /registry/secrets/default/prekleidi" | hexdump -C | grep mydata; +then + echo -e " unencrypted prekleidi Secret object found :)" +else + echo -e " /!\ no unencrypted prekleidi Secret object found!" +fi + +echo +echo -e " -> Checking a post kleidi deployment Secret" +# kubectl -n kube-system exec etcd-kleidi-vault-control-plane -- sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/etc/kubernetes/pki/etcd/ca.crt' ETCDCTL_CERT='/etc/kubernetes/pki/etcd/server.crt' ETCDCTL_KEY='/etc/kubernetes/pki/etcd/server.key' ETCDCTL_API=3 etcdctl get /registry/secrets/default/postkleidi" | hexdump -C + +if kubectl -n kube-system exec etcd-kleidi-vault-control-plane -- sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/etc/kubernetes/pki/etcd/ca.crt' ETCDCTL_CERT='/etc/kubernetes/pki/etcd/server.crt' ETCDCTL_KEY='/etc/kubernetes/pki/etcd/server.key' ETCDCTL_API=3 etcdctl get /registry/secrets/default/postkleidi" | hexdump -C | grep kms; +then + echo -e " encrypted postkleidi Secret object found :)" +else + echo -e " /!\ no encrypted postkleidi Secret object found!" + exit +fi + +echo +echo -e " -> Performing replace of prekleidi" +kubectl get secret prekleidi -o json | kubectl replace -f - + +echo -e " -> Checking a pre kleidi Secret replace" +# kubectl -n kube-system exec etcd-kleidi-vault-control-plane -- sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/etc/kubernetes/pki/etcd/ca.crt' ETCDCTL_CERT='/etc/kubernetes/pki/etcd/server.crt' ETCDCTL_KEY='/etc/kubernetes/pki/etcd/server.key' ETCDCTL_API=3 etcdctl get /registry/secrets/default/prekleidi" | hexdump -C + +if kubectl -n kube-system exec etcd-kleidi-vault-control-plane -- sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/etc/kubernetes/pki/etcd/ca.crt' ETCDCTL_CERT='/etc/kubernetes/pki/etcd/server.crt' ETCDCTL_KEY='/etc/kubernetes/pki/etcd/server.key' ETCDCTL_API=3 etcdctl get /registry/secrets/default/prekleidi" | hexdump -C |grep kms; +then + echo -e " encrypted prekleidi Secret object found :)" +else + echo -e " /!\ no encrypted prekleidi Secret object found!" +fi -# echo -e " -> Deploy kleidi kms plugin for vault" -# kubectl apply -f configuration/k8s/deploy/vault-pod-kleidi-kms.yaml +echo +echo -e " -> Cleaning any existing vault test env" +killall -9 vault ||true +echo +echo -e " -> Cleaning any existing kind test env" +kind delete cluster --name kleidi-vault -# echo -e " Cleaning any existing vault test env" -# killall -9 vault -# echo -e " Cleaning any existing kind test env" -# kind delete cluster --quiet --name kleidi-vault +echo +echo -e " -> Cleaning vault-encryption-config.yaml" +cp k8s/encryption/vault-encryption-config-bkp.yaml k8s/encryption/vault-encryption-config.yaml diff --git a/docs/architecture.md b/docs/architecture.md index 6679f9b..27272d9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,7 +21,7 @@ With successful ```initContainer```, the ```kleidi-kms-plugin``` container start * ```/var/lib(64)/softhsm/``` to access the token * ```/tmp/kleidi``` to create the gRPC socket -![kleidiv0.1](docs/images/kleidiv0.1.drawio.png) +![kleidiv0.1](images/kleidiv0.1.drawio.png) ## Why 1.29 or later? ***Stability!*** diff --git a/docs/security.md b/docs/security.md index 730ba14..2226c68 100644 --- a/docs/security.md +++ b/docs/security.md @@ -20,7 +20,7 @@ All data fields are encoded in base64 but not encrypted. The following diagram takes a 10,000-feet overview to explore the security exposures leading to a potential secret leaking/spilling: -![kleidi security exposures](docs/images/kledi-security_exposure.drawio.png) +![kleidi security exposures](images/kledi-security_exposure.drawio.png) * The secret comes from an external source and needs to be injected. * The base64 encoded secret will be ingested via the API server. diff --git a/docs/vault.md b/docs/vault.md index a61505d..5453f35 100644 --- a/docs/vault.md +++ b/docs/vault.md @@ -33,7 +33,7 @@ Development mode should NOT be used in production installations! Then export the Vault address (if not, it will default and fail on HTTPS): ``` -export VAULT_ADDR="http://:8200" +export VAULT_ADDR="http://0.0.0.0:8200" export VAULT_TOKEN="kleidi-demo" export VAULT_SKIP_VERIFY="true" ``` @@ -469,7 +469,7 @@ Expected output: Then let's encrypt the pre-deployment secret too: ``` -kubectl get secret prekleidi-secret -o json | /home/linuxbrew/.linuxbrew/bin/kubectl replace -f - +kubectl get secret prekleidi-secret -o json | kubectl replace -f - ``` Expected output: ```