From 13b1b7426b8d3a6bb97c190886b94968a1869d1d Mon Sep 17 00:00:00 2001 From: Miles Garnsey Date: Thu, 2 May 2024 15:54:13 +1000 Subject: [PATCH] First draft - use kind as a backing for envtests. Straighten out contexts. Addtnl error handling. Spawn kind clusters in parallel. Fix variable capture, try to get teardown working. Parallel cleanup. Make number of nodes per k8s cluster configurable, drop nodes to 1 per cluster for registration test. --- .../register/register_test.go | 93 ++++++--------- cmd/kubectl-k8ssandra/register/suite_test.go | 5 +- go.mod | 4 + go.sum | 19 ++++ internal/envtest/envtest.go | 26 +++++ internal/envtest/framework.go | 10 ++ internal/envtest/kind.go | 107 ++++++++++++++++++ internal/envtest/multi_envtest.go | 40 +++++++ pkg/registration/get_kubeconfig_test.go | 11 -- 9 files changed, 245 insertions(+), 70 deletions(-) create mode 100644 internal/envtest/kind.go delete mode 100644 pkg/registration/get_kubeconfig_test.go diff --git a/cmd/kubectl-k8ssandra/register/register_test.go b/cmd/kubectl-k8ssandra/register/register_test.go index bb5ce74..0b3d4fe 100644 --- a/cmd/kubectl-k8ssandra/register/register_test.go +++ b/cmd/kubectl-k8ssandra/register/register_test.go @@ -11,7 +11,9 @@ import ( configapi "github.com/k8ssandra/k8ssandra-operator/apis/config/v1beta1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/controller-runtime/pkg/client" @@ -19,21 +21,32 @@ import ( func TestRegister(t *testing.T) { require.New(t) - client1 := (*multiEnv)[0].GetClient("source-namespace") - client2 := (*multiEnv)[1].GetClient("dest-namespace") - - if err := client1.Create((*multiEnv)[0].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "source-namespace"}}); err != nil { - t.Fatal(err) - } - - if err := client2.Create((*multiEnv)[1].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dest-namespace"}}); err != nil { - t.Fatal(err) - } + client1, _ := client.New((*multiEnv)[0].RestConfig(), client.Options{}) + client2, _ := client.New((*multiEnv)[1].RestConfig(), client.Options{}) + ctx := context.Background() + require.Eventually(t, func() bool { + // It seems that at first, these clients may not be ready for use. By the time they can create a namespace they are known ready. + err1 := client1.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "source-namespace"}}) + if err1 != nil { + t.Log(err1) + if k8serrors.IsAlreadyExists(err1) { + err1 = nil + } + } + err2 := client2.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dest-namespace"}}) + if err2 != nil { + t.Log(err2) + if k8serrors.IsAlreadyExists(err2) { + err2 = nil + } + } + return err1 == nil && err2 == nil + }, time.Second*60, time.Second*5) buildDir := os.Getenv("BUILD_DIR") if buildDir == "" { _, b, _, _ := runtime.Caller(0) - buildDir = filepath.Join(filepath.Dir(b), "../../../build") + buildDir = filepath.Join(filepath.Dir(b), "../../build") } if _, err := os.Stat(buildDir); os.IsNotExist(err) { @@ -80,10 +93,9 @@ func TestRegister(t *testing.T) { SourceNamespace: "source-namespace", DestNamespace: "dest-namespace", ServiceAccount: "k8ssandra-operator", - Context: context.TODO(), + Context: ctx, DestinationName: "test-destination", } - ctx := context.Background() require.Eventually(t, func() bool { res := ex.RegisterCluster() @@ -92,51 +104,20 @@ func TestRegister(t *testing.T) { return true case res.IsError(): t.Log(res.GetError()) - if res.GetError().Error() == "no secret found for service account k8ssandra-operator" { - return true - } + return false + case res.IsRequeue(): + t.Log("requeing") + return false } return false - }, time.Second*30, time.Second*5) - - // This relies on a controller that is not running in the envtest. - - desiredSaSecret := &corev1.Secret{} - require.NoError(t, client1.Get(context.Background(), client.ObjectKey{Name: "k8ssandra-operator-secret", Namespace: "source-namespace"}, desiredSaSecret)) - patch := client.MergeFrom(desiredSaSecret.DeepCopy()) - desiredSaSecret.Data = map[string][]byte{ - "token": []byte("test-token"), - "ca.crt": []byte("test-ca"), - } - require.NoError(t, client1.Patch(ctx, desiredSaSecret, patch)) - - desiredSa := &corev1.ServiceAccount{} - require.NoError(t, client1.Get( - context.Background(), - client.ObjectKey{Name: "k8ssandra-operator", Namespace: "source-namespace"}, - desiredSa)) - - patch = client.MergeFrom(desiredSa.DeepCopy()) - desiredSa.Secrets = []corev1.ObjectReference{ - { - Name: "k8ssandra-operator-secret", - }, - } - require.NoError(t, client1.Patch(ctx, desiredSa, patch)) - - // Continue reconciliation + }, time.Second*60, time.Second*5) + sourceSecret := &corev1.Secret{} + // Ensure secret created. require.Eventually(t, func() bool { - res := ex.RegisterCluster() - switch { - case res.IsDone(): - return true - case res.IsError(): - t.Log(res.GetError()) - return false - } - return false - }, time.Second*3000, time.Second*5) + err := client1.Get(ctx, types.NamespacedName{Name: "k8ssandra-operator-secret", Namespace: "source-namespace"}, sourceSecret) + return err == nil + }, time.Second*60, time.Second*5) if err := configapi.AddToScheme(client2.Scheme()); err != nil { t.Fatal(err) @@ -161,11 +142,11 @@ func TestRegister(t *testing.T) { destKubeconfig := ClientConfigFromSecret(destSecret) require.Equal(t, - desiredSaSecret.Data["ca.crt"], + sourceSecret.Data["ca.crt"], destKubeconfig.Clusters["cluster"].CertificateAuthorityData) require.Equal(t, - string(desiredSaSecret.Data["token"]), + string(sourceSecret.Data["token"]), destKubeconfig.AuthInfos["cluster"].Token) } diff --git a/cmd/kubectl-k8ssandra/register/suite_test.go b/cmd/kubectl-k8ssandra/register/suite_test.go index 0211a89..90491be 100644 --- a/cmd/kubectl-k8ssandra/register/suite_test.go +++ b/cmd/kubectl-k8ssandra/register/suite_test.go @@ -12,8 +12,7 @@ var ( ) func TestMain(m *testing.M) { - // metrics.DefaultBindAddress = "0" This no longer appears to exist... - os.Exit(envtest.RunMulti(m, func(e *envtest.MultiK8sEnvironment) { + os.Exit(envtest.RunMultiKind(m, func(e *envtest.MultiK8sEnvironment) { multiEnv = e - }, 2)) + }, []int{1, 1})) } diff --git a/go.mod b/go.mod index d13c5fb..d3ad243 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect + github.com/alessio/shellescape v1.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -81,6 +82,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -124,6 +126,7 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect @@ -169,6 +172,7 @@ require ( k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kind v0.22.0 // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 92d732e..919e946 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -31,6 +32,8 @@ github.com/adutra/goalesce v0.0.0-20221124153206-5643f911003d h1:O47TZtmKaBkPubA github.com/adutra/goalesce v0.0.0-20221124153206-5643f911003d/go.mod h1:tRDczOw8/ipMRMBrO3kvAWxqQNnVXglAzhIqrXmKRAg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= @@ -82,6 +85,7 @@ github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -117,6 +121,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= @@ -211,6 +216,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de h1:6bMcLOeKoNo0+mTOb1ee3McF6CCKGixjLR3EDQY1Jik= github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -240,8 +247,10 @@ github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -348,6 +357,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= @@ -356,11 +366,15 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -407,6 +421,7 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -591,6 +606,7 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -638,11 +654,14 @@ sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeG sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.22.0 h1:z/+yr/azoOfzsfooqRsPw1wjJlqT/ukXP0ShkHwNlsI= +sigs.k8s.io/kind v0.22.0/go.mod h1:aBlbxg08cauDgZ612shr017/rZwqd7AS563FvpWKPVs= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index ed153ef..f61a71a 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -11,6 +11,7 @@ import ( "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/utils/ptr" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -37,6 +38,7 @@ type Environment struct { cancelManager context.CancelFunc Context context.Context Kubeconfig string + KindCluster KindManager } func NewEnvironment(ctx context.Context) *Environment { @@ -53,6 +55,30 @@ func NewEnvironment(ctx context.Context) *Environment { return env } +func NewKindEnvironment(ctx context.Context, cluster KindManager) *Environment { + err := cluster.CreateKindCluster() + if err != nil { + panic(err) + } + env := &Environment{} + env.env = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join(RootDir(), "testfiles", "crd"), + }, + UseExistingCluster: ptr.To(true), + ErrorIfCRDPathMissing: true, + Config: cluster.RestConfig, + } + ctx, cancel := context.WithCancel(ctx) + env.Context = ctx + env.cancelManager = cancel + env.Client = *cluster.client + env.Kubeconfig = cluster.KubeconfigLocation.Name() + env.env.Config = cluster.RestConfig + env.KindCluster = cluster + return env +} + func (e *Environment) GetClient(namespace string) client.Client { c, err := kubernetes.GetClientInNamespace(e.env.Config, namespace) if err != nil { diff --git a/internal/envtest/framework.go b/internal/envtest/framework.go index 320fa88..02c5687 100644 --- a/internal/envtest/framework.go +++ b/internal/envtest/framework.go @@ -1,6 +1,7 @@ package envtest import ( + "os" "path/filepath" "runtime" ) @@ -10,3 +11,12 @@ func RootDir() string { _, b, _, _ := runtime.Caller(0) return filepath.Join(filepath.Dir(b), "../..") } + +func GetBuildDir() string { + buildDir := os.Getenv("BUILD_DIR") + if buildDir == "" { + _, b, _, _ := runtime.Caller(0) + buildDir = filepath.Join(filepath.Dir(b), "../..") + } + return buildDir +} diff --git a/internal/envtest/kind.go b/internal/envtest/kind.go new file mode 100644 index 0000000..53beaa5 --- /dev/null +++ b/internal/envtest/kind.go @@ -0,0 +1,107 @@ +// mostly inspired by https://github.com/milesbxf/empathy/blob/master/cluster.go + +package envtest + +import ( + "log" + "os" + "time" + + "gopkg.in/yaml.v2" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + kindapi "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" + "sigs.k8s.io/kind/pkg/cluster" + kindcmd "sigs.k8s.io/kind/pkg/cmd" +) + +var kindConfig = ` +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker +- role: worker +` + +type KindManager struct { + ClusterName string + Nodes int + KubeconfigLocation *os.File + client *client.Client + RestConfig *rest.Config +} + +func (k *KindManager) CreateKindCluster() error { + log.Println("Creating kind cluster", k.ClusterName) + provider := cluster.NewProvider(cluster.ProviderWithLogger(kindcmd.NewLogger())) + kindConfig, err := setWorkerNodes(k.Nodes) + + if err != nil { + return err + } + + err = provider.Create( + k.ClusterName, + cluster.CreateWithNodeImage(""), + cluster.CreateWithRetain(false), + cluster.CreateWithWaitForReady(time.Duration(0)), + cluster.CreateWithKubeconfigPath(k.KubeconfigLocation.Name()), + cluster.CreateWithDisplayUsage(false), + cluster.CreateWithRawConfig([]byte(kindConfig)), + ) + if err != nil { + return err + } + config, err := clientcmd.BuildConfigFromFlags("", k.KubeconfigLocation.Name()) + if err != nil { + return err + } + kClient, err := client.New(config, client.Options{}) + if err != nil { + return err + } + k.client = &kClient + k.RestConfig = config + return err +} + +func (k *KindManager) TearDownKindCluster() error { + provider := cluster.NewProvider(cluster.ProviderWithLogger(kindcmd.NewLogger())) + return provider.Delete(k.ClusterName, k.KubeconfigLocation.Name()) +} + +func setWorkerNodes(workers int) ([]byte, error) { + cluster := &kindapi.Cluster{} + err := yaml.Unmarshal([]byte(kindConfig), cluster) + if err != nil { + return nil, err + } + cluster.Nodes = []kindapi.Node{ + {Role: kindapi.ControlPlaneRole}, + } + for i := 0; i < workers; i++ { + cluster.Nodes = append(cluster.Nodes, kindapi.Node{Role: kindapi.WorkerRole}) + } + return yaml.Marshal(cluster) +} + +func (k *KindManager) GetClient() (*client.Client, error) { + if k.client != nil { + return k.client, nil + } + + config, err := clientcmd.BuildConfigFromFlags("", k.KubeconfigLocation.Name()) + if err != nil { + return nil, err + } + + kclient, err := client.New(config, client.Options{}) + k.client = &kclient + if err != nil { + return nil, err + } + + return k.client, nil +} diff --git a/internal/envtest/multi_envtest.go b/internal/envtest/multi_envtest.go index e8da462..645de10 100644 --- a/internal/envtest/multi_envtest.go +++ b/internal/envtest/multi_envtest.go @@ -1,6 +1,9 @@ package envtest import ( + "os" + "strconv" + "sync" "testing" ctrl "sigs.k8s.io/controller-runtime" @@ -24,3 +27,40 @@ func RunMulti(m *testing.M, setupFunc func(e *MultiK8sEnvironment), numClusters exitCode := m.Run() return exitCode } + +func RunMultiKind(m *testing.M, setupFunc func(e *MultiK8sEnvironment), topology []int) (code int) { + e := make(MultiK8sEnvironment, len(topology)) + ctx := ctrl.SetupSignalHandler() + var wg sync.WaitGroup + for i := 0; i < len(topology); i++ { + cluster := KindManager{ + ClusterName: "cluster" + strconv.Itoa(i), + KubeconfigLocation: os.NewFile(0, GetBuildDir()+"/cluster"+strconv.Itoa(i)), + Nodes: topology[i], + } + wg.Add(1) + go func(i int) { + defer wg.Done() + e[i] = NewKindEnvironment(ctx, cluster) + e[i].Start() + }(i) + } + wg.Wait() + defer func() { + for i := 0; i < len(topology); i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + if err := e[i].KindCluster.TearDownKindCluster(); err != nil { + panic(err) + } + e[i].Stop() + }(i) + } + wg.Wait() + }() + + setupFunc(&e) + exitCode := m.Run() + return exitCode +} diff --git a/pkg/registration/get_kubeconfig_test.go b/pkg/registration/get_kubeconfig_test.go deleted file mode 100644 index 91694c3..0000000 --- a/pkg/registration/get_kubeconfig_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package registration - -import "testing" - -func TestGetKubeconfigFileLocation(t *testing.T) { - // TODO -} - -func TestGetClient(t *testing.T) { - // TODO -}