From 5fab6986613a71de16f160d2f2e757d2c7c9f402 Mon Sep 17 00:00:00 2001 From: Kellin McAvoy Date: Sat, 24 Oct 2020 18:17:59 -0500 Subject: [PATCH 1/2] refactor e2e directory --- e2e/Dockerfile | 3 --- e2e/entrypoint.sh | 2 +- e2e/framework/framework.go | 4 ++-- .../aws/deploy-localstack.sh} | 8 ++++---- .../aws/localstack.yaml} | 2 -- .../deploy-secret-manager.sh} | 14 +++++++------- .../default.yaml => secret-manager.yaml} | 0 e2e/tests/aws.go | 2 +- e2e/tests/smoke.go | 2 +- 9 files changed, 16 insertions(+), 21 deletions(-) rename e2e/{wait-for-localstack.sh => k8s/aws/deploy-localstack.sh} (79%) rename e2e/{localstack.deployment.yaml => k8s/aws/localstack.yaml} (92%) rename e2e/{wait-for-secret-manager.sh => k8s/deploy-secret-manager.sh} (73%) rename e2e/k8s/{helm-values/default.yaml => secret-manager.yaml} (100%) diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 5845bcf..8b7a706 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -25,9 +25,6 @@ COPY --from=builder /usr/local/bin/helm /usr/local/bin/ COPY entrypoint.sh /entrypoint.sh COPY e2e.test /e2e.test -COPY wait-for-secret-manager.sh /wait-for-secret-manager.sh -COPY wait-for-localstack.sh /wait-for-localstack.sh COPY k8s /k8s -COPY localstack.deployment.yaml /localstack.deployment.yaml CMD [ "/entrypoint.sh" ] diff --git a/e2e/entrypoint.sh b/e2e/entrypoint.sh index 2b6b243..4dcefeb 100755 --- a/e2e/entrypoint.sh +++ b/e2e/entrypoint.sh @@ -39,7 +39,7 @@ ginkgo_args=( "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}" "-r" "-v" - "-timeout=45m" + "-timeout=30m" ) echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}" diff --git a/e2e/framework/framework.go b/e2e/framework/framework.go index 0850473..2b2c376 100644 --- a/e2e/framework/framework.go +++ b/e2e/framework/framework.go @@ -130,7 +130,7 @@ func (f *Framework) AfterEach() { func (f *Framework) newSecretManager() error { ginkgo.By("launching secret-manager") //nolint:gosec - cmd := exec.Command("/wait-for-secret-manager.sh", f.Namespace, f.HelmValues, fmt.Sprintf("secret-manager-%s", f.BaseName)) + cmd := exec.Command("/k8s/deploy-secret-manager.sh", f.Namespace, f.HelmValues, fmt.Sprintf("secret-manager-%s", f.BaseName)) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("unexpected error creating secret-manager: %v.\nLogs:\n%v", err, string(out)) @@ -153,7 +153,7 @@ func (f *Framework) deleteSecretManager() error { // NewLocalstack deploys a fresh localstack instance into the specified namespace func (f *Framework) NewLocalstack(namespace string) error { ginkgo.By("launching localstack") - cmd := exec.Command("/wait-for-localstack.sh", namespace, f.HelmValues) + cmd := exec.Command("/k8s/aws/deploy-localstack.sh", namespace, f.HelmValues) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("unexpected error creating localstack: %v.\nLogs:\n%v", err, string(out)) diff --git a/e2e/wait-for-localstack.sh b/e2e/k8s/aws/deploy-localstack.sh similarity index 79% rename from e2e/wait-for-localstack.sh rename to e2e/k8s/aws/deploy-localstack.sh index ef6caa6..d1202af 100755 --- a/e2e/wait-for-localstack.sh +++ b/e2e/k8s/aws/deploy-localstack.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. set -e -if ! [ -z $DEBUG ]; then +if [ -n "$DEBUG" ]; then set -x fi @@ -30,9 +30,9 @@ function on_exit { test $error_code == 0 && return; echo "Obtaining secret-manager pod logs..." - kubectl logs -l app=localstack -n $NAMESPACE + kubectl logs --selector=app=localstack -n "$NAMESPACE" } trap on_exit EXIT -kubectl apply -n $NAMESPACE -f ${DIR}/localstack.deployment.yaml -kubectl rollout status -n $NAMESPACE deploy/localstack +kubectl apply -n "$NAMESPACE" -f "$DIR/localstack.yaml" +kubectl wait pod -n "$NAMESPACE" --for=condition=Ready --selector=app=localstack --timeout=5m diff --git a/e2e/localstack.deployment.yaml b/e2e/k8s/aws/localstack.yaml similarity index 92% rename from e2e/localstack.deployment.yaml rename to e2e/k8s/aws/localstack.yaml index 3a923a7..154bac0 100644 --- a/e2e/localstack.deployment.yaml +++ b/e2e/k8s/aws/localstack.yaml @@ -39,8 +39,6 @@ kind: Service metadata: name: localstack spec: - # selector tells Kubernetes what Deployment this Service - # belongs to selector: app: localstack ports: diff --git a/e2e/wait-for-secret-manager.sh b/e2e/k8s/deploy-secret-manager.sh similarity index 73% rename from e2e/wait-for-secret-manager.sh rename to e2e/k8s/deploy-secret-manager.sh index 064662c..f0906dc 100755 --- a/e2e/wait-for-secret-manager.sh +++ b/e2e/k8s/deploy-secret-manager.sh @@ -15,7 +15,7 @@ # limitations under the License. set -e -if ! [ -z $DEBUG ]; then +if [ -n "$DEBUG" ]; then set -x fi @@ -33,14 +33,14 @@ function on_exit { test $error_code == 0 && return; echo "Obtaining secret-manager pod logs..." - kubectl logs -l app=secret-manager -n ${NAMESPACE} + kubectl logs -l app=secret-manager -n "$NAMESPACE" } trap on_exit EXIT -echo "Helm values file ${HELM_VALUES_FILE} is being used for namespace ${NAMESPACE}" +echo "Helm values file $HELM_VALUES_FILE is being used for namespace $NAMESPACE" -helm install ${RELEASE_NAME} ${DIR}/k8s/deploy/charts/secret-manager \ - --namespace=${NAMESPACE} \ - --values "${DIR}/k8s/helm-values/${HELM_VALUES_FILE}.yaml" \ +helm install "$RELEASE_NAME" "$DIR/deploy/charts/secret-manager" \ + --namespace="$NAMESPACE" \ + --values "$DIR/$HELM_VALUES_FILE.yaml" \ --wait \ - --timeout=10m + --timeout=5m diff --git a/e2e/k8s/helm-values/default.yaml b/e2e/k8s/secret-manager.yaml similarity index 100% rename from e2e/k8s/helm-values/default.yaml rename to e2e/k8s/secret-manager.yaml diff --git a/e2e/tests/aws.go b/e2e/tests/aws.go index e21749c..6d66661 100644 --- a/e2e/tests/aws.go +++ b/e2e/tests/aws.go @@ -34,7 +34,7 @@ import ( ) var _ = ginkgo.Describe("[aws]", func() { - f := framework.NewDefaultFramework("aws", "default") + f := framework.NewDefaultFramework("aws", "secret-manager") ginkgo.BeforeEach(func() { err := f.NewLocalstack(f.Namespace) diff --git a/e2e/tests/smoke.go b/e2e/tests/smoke.go index 122528b..0cfc67c 100644 --- a/e2e/tests/smoke.go +++ b/e2e/tests/smoke.go @@ -27,7 +27,7 @@ import ( ) var _ = ginkgo.Describe("[smoke]", func() { - f := framework.NewDefaultFramework("smoke", "default") + f := framework.NewDefaultFramework("smoke", "secret-manager") ginkgo.It("should expose metrics", func() { po, err := framework.WaitForSMPod(f.Namespace, f.KubeClient) From a931ca5dd1c3e69953b8e56e3701a59b964d9f49 Mon Sep 17 00:00:00 2001 From: Kellin McAvoy Date: Mon, 26 Oct 2020 21:20:41 -0500 Subject: [PATCH 2/2] feat: add initial vault tests --- e2e/Dockerfile | 3 +- e2e/addon/vault/util.go | 156 ++++++++++++++++++ e2e/addon/vault/vault.go | 155 +++++++++++++++++ e2e/entrypoint.sh | 2 +- e2e/framework/aws.go | 20 ++- e2e/framework/framework.go | 11 -- .../aws/{deploy-localstack.sh => deploy.sh} | 2 +- e2e/k8s/aws/localstack.yaml | 1 - e2e/k8s/vault/vault.yaml | 154 +++++++++++++++++ e2e/run.sh | 5 +- e2e/tests/aws.go | 1 - e2e/tests/vault.go | 121 ++++++++++++++ 12 files changed, 612 insertions(+), 19 deletions(-) create mode 100644 e2e/addon/vault/util.go create mode 100644 e2e/addon/vault/vault.go rename e2e/k8s/aws/{deploy-localstack.sh => deploy.sh} (95%) create mode 100644 e2e/k8s/vault/vault.yaml create mode 100644 e2e/tests/vault.go diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 8b7a706..c4ae531 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -17,7 +17,8 @@ RUN apk add -U --no-cache \ curl \ tzdata \ libc6-compat \ - openssl + openssl \ + jq COPY --from=builder /go/bin/ginkgo /usr/local/bin/ COPY --from=builder /usr/local/bin/kubectl /usr/local/bin/ diff --git a/e2e/addon/vault/util.go b/e2e/addon/vault/util.go new file mode 100644 index 0000000..86a4c90 --- /dev/null +++ b/e2e/addon/vault/util.go @@ -0,0 +1,156 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vault + +import ( + "context" + "errors" + "fmt" + "time" + + vault "github.com/hashicorp/vault/api" + + smmeta "github.com/itscontained/secret-manager/pkg/apis/meta/v1" + smv1alpha1 "github.com/itscontained/secret-manager/pkg/apis/secretmanager/v1alpha1" + + corev1 "k8s.io/api/core/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" +) + +func (v *Vault) CreateSecretStoreVaultKubeAuth() (*smv1alpha1.SecretStoreSpec, error) { + randomSuffix := uuid.NewUUID() + v.kubePath = fmt.Sprintf("kubernetes-%s", randomSuffix) + v.kubeRole = fmt.Sprintf("role-%s", randomSuffix) + + // Create Kubernetes auth backend + err := v.vaultClient.Sys().EnableAuthWithOptions(v.kubePath, &vault.EnableAuthOptions{ + Type: "kubernetes", + }) + if err != nil { + return nil, err + } + + // Get token reviewer jwt for vault + tokenReviewerAccount := &corev1.ServiceAccount{} + v.KubeClient.Get(context.Background(), types.NamespacedName{ + Namespace: v.Namespace, + Name: serviceAccountName, + }, tokenReviewerAccount) + + if len(tokenReviewerAccount.Secrets) == 0 { + return nil, errors.New("vault serviceaccount has no associated secret") + } + tokenSecret := &corev1.Secret{} + v.KubeClient.Get(context.Background(), types.NamespacedName{ + Namespace: v.Namespace, + Name: tokenReviewerAccount.Secrets[0].Name, + }, tokenSecret) + + kubeReq := v.vaultClient.NewRequest("POST", fmt.Sprintf("/v1/auth/%s/config", v.kubePath)) + kubeData := map[string]interface{}{ + "kubernetes_host": "https://kubernetes.default", + "token_reviewer_jwt": string(tokenSecret.Data["token"]), + } + kubeReq.SetJSONBody(kubeData) + _, err = v.vaultClient.RawRequest(kubeReq) + if err != nil { + return nil, err + } + + // Create vault role for Kubernetes auth + svcAccountName := fmt.Sprintf("kv-reader-%s", randomSuffix) + req := v.vaultClient.NewRequest("POST", fmt.Sprintf("/v1/auth/%s/role/%s", v.kubePath, v.kubeRole)) + roleData := map[string]interface{}{ + "bound_service_account_names": []string{svcAccountName}, + "bound_service_account_namespaces": []string{v.Namespace}, + "token_policies": []string{v.secretRole}, + } + req.SetJSONBody(roleData) + _, err = v.vaultClient.RawRequest(req) + if err != nil { + return nil, err + } + + err = v.KubeClient.Create(context.Background(), &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcAccountName, + Namespace: v.Namespace, + }, + }) + if err != nil { + return nil, err + } + + svcAccountCreated := &corev1.ServiceAccount{} + svcAccountKey := types.NamespacedName{ + Name: svcAccountName, + Namespace: v.Namespace, + } + + err = try(3, time.Second*2, func() error { + err = v.KubeClient.Get(context.Background(), svcAccountKey, svcAccountCreated) + if err != nil { + return err + } + + if len(svcAccountCreated.Secrets) < 1 { + return errors.New("created service account has no token secret") + } + return nil + }) + if err != nil { + return nil, err + } + + return &smv1alpha1.SecretStoreSpec{ + Vault: &smv1alpha1.VaultStore{ + Server: v.Host, + Path: v.kvPath, + Auth: smv1alpha1.VaultAuth{ + Kubernetes: &smv1alpha1.VaultKubernetesAuth{ + Path: v.kubePath, + Role: v.kubeRole, + SecretRef: &smmeta.SecretKeySelector{ + LocalObjectReference: smmeta.LocalObjectReference{ + Name: svcAccountCreated.Secrets[0].Name, + }, + Key: "token", + }, + }, + }, + }, + }, nil +} + +// try attempts a function for n retries, with pollPeriod waiting in-between +// when the function returns no error, nil is return. Error is returned +// after retries with error wrapped. +func try(retries int, pollPeriod time.Duration, f func() error) error { + attempt := 0 + var err error + for attempt < retries { + err = f() + if err == nil { + return nil + } + time.Sleep(pollPeriod) + attempt++ + } + + return fmt.Errorf("retry attempts failed: %w", err) +} diff --git a/e2e/addon/vault/vault.go b/e2e/addon/vault/vault.go new file mode 100644 index 0000000..83d1cdd --- /dev/null +++ b/e2e/addon/vault/vault.go @@ -0,0 +1,155 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vault + +import ( + "context" + "errors" + "fmt" + "os/exec" + "path/filepath" + "strings" + "time" + + vault "github.com/hashicorp/vault/api" + + rbacv1 "k8s.io/api/rbac/v1" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + selector string = "app=vault" + serviceAddress string = "http://vault.%s:8200" + serviceAccountName string = "vault" + defaultToken string = "root" + secretPolicy string = ` +path "%s/*" { + capabilities = ["read"] +} +` +) + +// Vault describes the configuration details for an instance of Vault +// deployed to the test cluster +type Vault struct { + // Kubectl is the path to kubectl + Kubectl string + + // Namespace is the namespace to deploy Vault into + Namespace string + + // KubeClient is a configured Kubernetes clientset for addons to use. + KubeClient ctrlclient.Client + + // Host is the hostname that can be used to connect to Vault + Host string + + // BasePath is root of deployment manifests + BasePath string + + vaultClient *vault.Client + kvPath string + secretRole string + kubePath string + kubeRole string +} + +func (v *Vault) Setup() error { + if v.Kubectl == "" { + return errors.New("kubectl must be set") + } + + if v.Namespace == "" { + return errors.New("namespace must be set") + } + v.kvPath = "secret" + + err := Run(fmt.Sprintf("%s apply --timeout 2m -f %s -n %s", v.Kubectl, filepath.Join(v.BasePath, "/k8s/vault/vault.yaml"), v.Namespace)) + if err != nil { + return err + } + // create cluster resources because namespace is not known beforehand + err = v.createClusterResources() + if err != nil { + return err + } + + time.Sleep(10 * time.Second) + err = Run(fmt.Sprintf("%s wait pod --for=condition=Ready --timeout 2m -n %s -l %s", v.Kubectl, v.Namespace, selector)) + if err != nil { + return err + } + v.Host = fmt.Sprintf(serviceAddress, v.Namespace) + vaultConfig := vault.DefaultConfig() + vaultConfig.Address = v.Host + v.vaultClient, err = vault.NewClient(vaultConfig) + if err != nil { + return err + } + v.vaultClient.SetToken(defaultToken) + + v.secretRole = "kv-reader" + err = v.vaultClient.Sys().PutPolicy(v.secretRole, fmt.Sprintf(secretPolicy, v.kvPath)) + return err +} + +func Run(cmd string) error { + splitCmd := strings.Split(cmd, " ") + c := exec.Command(splitCmd[0], splitCmd[1:]...) + out, err := c.CombinedOutput() + if err != nil { + return fmt.Errorf("error executing command: %s\nerr: %w", string(out), err) + } + return nil +} + +func (v *Vault) CreateSecret(name string, secret map[string]string) error { + vaultPath := fmt.Sprintf("%s/data/%s", v.kvPath, name) + vaultData := make(map[string]interface{}, 1) + vaultData["data"] = secret + _, err := v.vaultClient.Logical().Write(vaultPath, vaultData) + return err +} + +func (v *Vault) createClusterResources() error { + vaultCRB := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("vault-auth-%s", v.Namespace), + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "system:auth-delegator", + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: serviceAccountName, + Namespace: v.Namespace, + }, + }, + } + + err := v.KubeClient.Create(context.Background(), vaultCRB) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + } + return nil +} diff --git a/e2e/entrypoint.sh b/e2e/entrypoint.sh index 4dcefeb..5d530d6 100755 --- a/e2e/entrypoint.sh +++ b/e2e/entrypoint.sh @@ -39,7 +39,7 @@ ginkgo_args=( "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}" "-r" "-v" - "-timeout=30m" + "-timeout=5m" ) echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}" diff --git a/e2e/framework/aws.go b/e2e/framework/aws.go index cbb86ef..ba17162 100644 --- a/e2e/framework/aws.go +++ b/e2e/framework/aws.go @@ -17,11 +17,18 @@ package framework import ( "context" "fmt" + "os/exec" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/endpoints" "github.com/aws/aws-sdk-go-v2/aws/external" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + + "github.com/onsi/ginkgo" +) + +const ( + localstackDeploy = "/k8s/aws/deploy.sh" ) // CreateAWSSecretsManagerSecret creates a sm secret with the given value @@ -42,7 +49,7 @@ func CreateAWSSecretsManagerSecret(namespace, name, secret string) error { return err } -// localResolver resolves endpoints to +// localResolver resolves endpoints to e2e localstack type localResolver struct { endpoints.Resolver namespace string @@ -54,3 +61,14 @@ func (r *localResolver) ResolveEndpoint(service, region string) (aws.Endpoint, e URL: fmt.Sprintf("http://localstack.%s", r.namespace), }, nil } + +// NewLocalstack deploys a fresh localstack instance into the specified namespace +func (f *Framework) NewLocalstack(namespace string) error { + ginkgo.By("launching localstack") + cmd := exec.Command(localstackDeploy, namespace, f.HelmValues) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("unexpected error creating localstack: %v.\nLogs:\n%v", err, string(out)) + } + return nil +} diff --git a/e2e/framework/framework.go b/e2e/framework/framework.go index 2b2c376..212c19c 100644 --- a/e2e/framework/framework.go +++ b/e2e/framework/framework.go @@ -149,14 +149,3 @@ func (f *Framework) deleteSecretManager() error { } return nil } - -// NewLocalstack deploys a fresh localstack instance into the specified namespace -func (f *Framework) NewLocalstack(namespace string) error { - ginkgo.By("launching localstack") - cmd := exec.Command("/k8s/aws/deploy-localstack.sh", namespace, f.HelmValues) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("unexpected error creating localstack: %v.\nLogs:\n%v", err, string(out)) - } - return nil -} diff --git a/e2e/k8s/aws/deploy-localstack.sh b/e2e/k8s/aws/deploy.sh similarity index 95% rename from e2e/k8s/aws/deploy-localstack.sh rename to e2e/k8s/aws/deploy.sh index d1202af..1711936 100755 --- a/e2e/k8s/aws/deploy-localstack.sh +++ b/e2e/k8s/aws/deploy.sh @@ -29,7 +29,7 @@ function on_exit { test $error_code == 0 && return; - echo "Obtaining secret-manager pod logs..." + echo "Obtaining aws localstack pod logs..." kubectl logs --selector=app=localstack -n "$NAMESPACE" } trap on_exit EXIT diff --git a/e2e/k8s/aws/localstack.yaml b/e2e/k8s/aws/localstack.yaml index 154bac0..2493cde 100644 --- a/e2e/k8s/aws/localstack.yaml +++ b/e2e/k8s/aws/localstack.yaml @@ -44,4 +44,3 @@ spec: ports: - port: 80 targetPort: edge ---- diff --git a/e2e/k8s/vault/vault.yaml b/e2e/k8s/vault/vault.yaml new file mode 100644 index 0000000..8e3442c --- /dev/null +++ b/e2e/k8s/vault/vault.yaml @@ -0,0 +1,154 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault + labels: + app: vault +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: vault + labels: + app: vault +spec: + serviceName: vault-internal + podManagementPolicy: Parallel + replicas: 1 + updateStrategy: + type: OnDelete + selector: + matchLabels: + app: vault + component: server + template: + metadata: + labels: + app: vault + component: server + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: vault + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 100 + fsGroup: 1000 + volumes: + - name: home + emptyDir: {} + containers: + - name: vault + image: vault:1.5.2 + imagePullPolicy: IfNotPresent + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: VAULT_K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VAULT_K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: VAULT_ADDR + value: "http://127.0.0.1:8200" + - name: VAULT_API_ADDR + value: "http://$(POD_IP):8200" + - name: SKIP_CHOWN + value: "true" + - name: SKIP_SETCAP + value: "true" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VAULT_CLUSTER_ADDR + value: "https://$(HOSTNAME).vault-internal:8201" + - name: HOME + value: "/home/vault" + - name: VAULT_DEV_ROOT_TOKEN_ID + value: "root" + volumeMounts: + - name: home + mountPath: /home/vault + ports: + - containerPort: 8200 + name: http + - containerPort: 8201 + name: https-internal + - containerPort: 8202 + name: http-rep + readinessProbe: + # Check status; unsealed vault servers return 0 + # The exit code reflects the seal status: + # 0 - unsealed + # 1 - error + # 2 - sealed + exec: + command: ["/bin/sh", "-ec", "vault status -tls-skip-verify"] + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 5 + lifecycle: + # Vault container doesn't receive SIGTERM from Kubernetes + # and after the grace period ends, Kube sends SIGKILL. This + # causes issues with graceful shutdowns such as deregistering itself + # from Consul (zombie services). + preStop: + exec: + command: [ + "/bin/sh", "-c", + # Adding a sleep here to give the pod eviction a + # chance to propagate, so requests will not be made + # to this pod while it's terminating + "sleep 5 && kill -SIGTERM $(pidof vault)", + ] +--- +apiVersion: v1 +kind: Service +metadata: + name: vault-internal + labels: + app: vault +spec: + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http + port: 8200 + targetPort: 8200 + - name: https-internal + port: 8201 + targetPort: 8201 + selector: + app: vault +--- +apiVersion: v1 +kind: Service +metadata: + name: vault + labels: + app: vault +spec: + # We want the servers to become available even if they're not ready + # since this DNS is also used for join operations. + publishNotReadyAddresses: true + ports: + - name: http + port: 8200 + targetPort: 8200 + - name: https-internal + port: 8201 + targetPort: 8201 + selector: + app: vault diff --git a/e2e/run.sh b/e2e/run.sh index 8abf517..cb61bd7 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -48,9 +48,10 @@ echo -e "Starting the e2e test pod" FOCUS=${FOCUS:-.*} export FOCUS -kubectl run --rm \ +kubectl run e2e \ + --rm \ --attach \ --restart=Never \ --env="FOCUS=${FOCUS}" \ --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "secret-manager-e2e"}}' \ - e2e --image=local/secret-manager-e2e:test + --image=local/secret-manager-e2e:test diff --git a/e2e/tests/aws.go b/e2e/tests/aws.go index 6d66661..1d057de 100644 --- a/e2e/tests/aws.go +++ b/e2e/tests/aws.go @@ -21,7 +21,6 @@ import ( smmeta "github.com/itscontained/secret-manager/pkg/apis/meta/v1" smv1alpha1 "github.com/itscontained/secret-manager/pkg/apis/secretmanager/v1alpha1" - // use dot imports "github.com/onsi/ginkgo" "github.com/onsi/gomega" diff --git a/e2e/tests/vault.go b/e2e/tests/vault.go new file mode 100644 index 0000000..b9b7201 --- /dev/null +++ b/e2e/tests/vault.go @@ -0,0 +1,121 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "context" + + vaultaddon "github.com/itscontained/secret-manager/e2e/addon/vault" + "github.com/itscontained/secret-manager/e2e/framework" + smmeta "github.com/itscontained/secret-manager/pkg/apis/meta/v1" + smv1alpha1 "github.com/itscontained/secret-manager/pkg/apis/secretmanager/v1alpha1" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/stretchr/testify/assert" + + corev1 "k8s.io/api/core/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("[vault]", func() { + f := framework.NewDefaultFramework("vault", "secret-manager") + // TODO: make framework addon setup more generic + vaultDeploy := vaultaddon.Vault{ + Kubectl: "kubectl", + Namespace: f.Namespace, + KubeClient: f.KubeClient, + } + + BeforeEach(func() { + err := vaultDeploy.Setup() + assert.Nil(GinkgoT(), err, "creating vault") + }) + + It("should sync secrets", func() { + // create secret + secretName := "teamA/example-service" + secret := map[string]string{ + "new-secret": "test-value123", + } + err := vaultDeploy.CreateSecret(secretName, secret) + Expect(err).ToNot(HaveOccurred()) + + key := types.NamespacedName{ + Name: "vault-secret1", + Namespace: f.Namespace, + } + + storeSpec, err := vaultDeploy.CreateSecretStoreVaultKubeAuth() + By("Setup Vault Kubernetes auth backend successfully") + Expect(err).Should(Succeed()) + // create store + store := &smv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault", + Namespace: f.Namespace, + }, + Spec: *storeSpec, + } + By("Creating the Vault SecretStore successfully") + Expect(f.KubeClient.Create(context.Background(), store)).Should(Succeed()) + + // create ES + By("Creating the ExternalSecret successfully") + Expect(f.KubeClient.Create(context.Background(), &smv1alpha1.ExternalSecret{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: f.Namespace, + }, + Spec: smv1alpha1.ExternalSecretSpec{ + StoreRef: smv1alpha1.ObjectReference{ + Name: store.Name, + Kind: smv1alpha1.SecretStoreKind, + }, + Data: []smv1alpha1.KeyReference{ + { + SecretKey: "user-a", + RemoteRef: smv1alpha1.RemoteReference{ + Name: secretName, + Property: smmeta.String("new-secret"), + }, + }, + }, + }, + })).Should(Succeed()) + + // wait for secret to appear + fetched := &smv1alpha1.ExternalSecret{} + Eventually(func() bool { + By("Fetching the ExternalSecret successfully") + Expect(f.KubeClient.Get(context.Background(), key, fetched)).Should(Succeed()) + By("Checking the status condition") + fetchedCond := fetched.Status.GetCondition(smmeta.TypeReady) + return fetchedCond.Matches(smmeta.Available()) + }, framework.DefaultTimeout, framework.Poll).Should(BeTrue(), "The ExternalSecret should have a ready condition") + + fetchedSecret := &corev1.Secret{} + Eventually(func() map[string][]byte { + By("Fetching the Secret successfully") + Expect(f.KubeClient.Get(context.Background(), key, fetchedSecret)).Should(Succeed()) + return fetchedSecret.Data + }, framework.DefaultTimeout, framework.Poll).Should(Equal(map[string][]byte{ + "user-a": []byte("test-value123"), + }), "The generated secret should be created") + }) +})