From 460cf96907dc3980b3a5d9ea5d0c0233e1778d4b Mon Sep 17 00:00:00 2001 From: Miles Garnsey Date: Thu, 2 May 2024 15:54:13 +1000 Subject: [PATCH 1/3] First draft - use kind as a backing for envtests. Straighten out contexts. Addtnl error handling. Spawn kind clusters in parallel. Fix variable capture, try to get teardown working. Parallel cleanup. Make number of nodes per k8s cluster configurable, drop nodes to 1 per cluster for registration test. --- .../register/register_test.go | 28 ++++- cmd/kubectl-k8ssandra/register/suite_test.go | 5 +- go.mod | 4 + go.sum | 19 ++++ internal/envtest/envtest.go | 29 ++++- internal/envtest/kind.go | 107 ++++++++++++++++++ internal/envtest/multi_envtest.go | 40 +++++++ internal/envtest/root_dir.go | 10 ++ 8 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 internal/envtest/kind.go diff --git a/cmd/kubectl-k8ssandra/register/register_test.go b/cmd/kubectl-k8ssandra/register/register_test.go index 08f48c7..16cb672 100644 --- a/cmd/kubectl-k8ssandra/register/register_test.go +++ b/cmd/kubectl-k8ssandra/register/register_test.go @@ -11,6 +11,7 @@ import ( configapi "github.com/k8ssandra/k8ssandra-operator/apis/config/v1beta1" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -19,10 +20,27 @@ import ( func TestRegister(t *testing.T) { require := require.New(t) - client1 := (*multiEnv)[0].GetClientInNamespace("source-namespace") - client2 := (*multiEnv)[1].GetClientInNamespace("dest-namespace") - require.NoError(client1.Create((*multiEnv)[0].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "source-namespace"}})) - require.NoError(client2.Create((*multiEnv)[1].Context, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dest-namespace"}})) + client1, _ := client.New((*multiEnv)[0].RestConfig(), client.Options{}) + client2, _ := client.New((*multiEnv)[1].RestConfig(), client.Options{}) + ctx := context.Background() + require.Eventually(func() bool { + // It seems that at first, these clients may not be ready for use. By the time they can create a namespace they are known ready. + err1 := client1.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "source-namespace"}}) + if err1 != nil { + t.Log(err1) + if k8serrors.IsAlreadyExists(err1) { + err1 = nil + } + } + err2 := client2.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "dest-namespace"}}) + if err2 != nil { + t.Log(err2) + if k8serrors.IsAlreadyExists(err2) { + err2 = nil + } + } + return err1 == nil && err2 == nil + }, time.Second*60, time.Second*5) testDir, err := os.MkdirTemp("", "k8ssandra-operator-test-****") require.NoError(err) @@ -59,7 +77,7 @@ func TestRegister(t *testing.T) { SourceNamespace: "source-namespace", DestNamespace: "dest-namespace", ServiceAccount: "k8ssandra-operator", - Context: context.TODO(), + Context: ctx, DestinationName: "test-destination", } ctx := context.Background() diff --git a/cmd/kubectl-k8ssandra/register/suite_test.go b/cmd/kubectl-k8ssandra/register/suite_test.go index 0211a89..90491be 100644 --- a/cmd/kubectl-k8ssandra/register/suite_test.go +++ b/cmd/kubectl-k8ssandra/register/suite_test.go @@ -12,8 +12,7 @@ var ( ) func TestMain(m *testing.M) { - // metrics.DefaultBindAddress = "0" This no longer appears to exist... - os.Exit(envtest.RunMulti(m, func(e *envtest.MultiK8sEnvironment) { + os.Exit(envtest.RunMultiKind(m, func(e *envtest.MultiK8sEnvironment) { multiEnv = e - }, 2)) + }, []int{1, 1})) } diff --git a/go.mod b/go.mod index d13c5fb..d3ad243 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect + github.com/alessio/shellescape v1.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -81,6 +82,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -124,6 +126,7 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect @@ -169,6 +172,7 @@ require ( k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kind v0.22.0 // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 92d732e..919e946 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -31,6 +32,8 @@ github.com/adutra/goalesce v0.0.0-20221124153206-5643f911003d h1:O47TZtmKaBkPubA github.com/adutra/goalesce v0.0.0-20221124153206-5643f911003d/go.mod h1:tRDczOw8/ipMRMBrO3kvAWxqQNnVXglAzhIqrXmKRAg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= @@ -82,6 +85,7 @@ github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -117,6 +121,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= @@ -211,6 +216,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de h1:6bMcLOeKoNo0+mTOb1ee3McF6CCKGixjLR3EDQY1Jik= github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -240,8 +247,10 @@ github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -348,6 +357,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= @@ -356,11 +366,15 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -407,6 +421,7 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -591,6 +606,7 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -638,11 +654,14 @@ sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeG sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.22.0 h1:z/+yr/azoOfzsfooqRsPw1wjJlqT/ukXP0ShkHwNlsI= +sigs.k8s.io/kind v0.22.0/go.mod h1:aBlbxg08cauDgZ612shr017/rZwqd7AS563FvpWKPVs= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index 120c290..dde3a5b 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -11,6 +11,7 @@ import ( "github.com/k8ssandra/k8ssandra-client/pkg/kubernetes" k8ssandrataskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/utils/ptr" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -36,6 +37,8 @@ type Environment struct { env *envtest.Environment cancelManager context.CancelFunc Context context.Context + Kubeconfig string + KindCluster KindManager } func NewEnvironment(ctx context.Context) *Environment { @@ -52,7 +55,31 @@ func NewEnvironment(ctx context.Context) *Environment { return env } -func (e *Environment) GetClientInNamespace(namespace string) client.Client { +func NewKindEnvironment(ctx context.Context, cluster KindManager) *Environment { + err := cluster.CreateKindCluster() + if err != nil { + panic(err) + } + env := &Environment{} + env.env = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join(RootDir(), "testfiles", "crd"), + }, + UseExistingCluster: ptr.To(true), + ErrorIfCRDPathMissing: true, + Config: cluster.RestConfig, + } + ctx, cancel := context.WithCancel(ctx) + env.Context = ctx + env.cancelManager = cancel + env.Client = *cluster.client + env.Kubeconfig = cluster.KubeconfigLocation.Name() + env.env.Config = cluster.RestConfig + env.KindCluster = cluster + return env +} + +func (e *Environment) GetClient(namespace string) client.Client { c, err := kubernetes.GetClientInNamespace(e.env.Config, namespace) if err != nil { panic(err) diff --git a/internal/envtest/kind.go b/internal/envtest/kind.go new file mode 100644 index 0000000..53beaa5 --- /dev/null +++ b/internal/envtest/kind.go @@ -0,0 +1,107 @@ +// mostly inspired by https://github.com/milesbxf/empathy/blob/master/cluster.go + +package envtest + +import ( + "log" + "os" + "time" + + "gopkg.in/yaml.v2" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + kindapi "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" + "sigs.k8s.io/kind/pkg/cluster" + kindcmd "sigs.k8s.io/kind/pkg/cmd" +) + +var kindConfig = ` +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker +- role: worker +` + +type KindManager struct { + ClusterName string + Nodes int + KubeconfigLocation *os.File + client *client.Client + RestConfig *rest.Config +} + +func (k *KindManager) CreateKindCluster() error { + log.Println("Creating kind cluster", k.ClusterName) + provider := cluster.NewProvider(cluster.ProviderWithLogger(kindcmd.NewLogger())) + kindConfig, err := setWorkerNodes(k.Nodes) + + if err != nil { + return err + } + + err = provider.Create( + k.ClusterName, + cluster.CreateWithNodeImage(""), + cluster.CreateWithRetain(false), + cluster.CreateWithWaitForReady(time.Duration(0)), + cluster.CreateWithKubeconfigPath(k.KubeconfigLocation.Name()), + cluster.CreateWithDisplayUsage(false), + cluster.CreateWithRawConfig([]byte(kindConfig)), + ) + if err != nil { + return err + } + config, err := clientcmd.BuildConfigFromFlags("", k.KubeconfigLocation.Name()) + if err != nil { + return err + } + kClient, err := client.New(config, client.Options{}) + if err != nil { + return err + } + k.client = &kClient + k.RestConfig = config + return err +} + +func (k *KindManager) TearDownKindCluster() error { + provider := cluster.NewProvider(cluster.ProviderWithLogger(kindcmd.NewLogger())) + return provider.Delete(k.ClusterName, k.KubeconfigLocation.Name()) +} + +func setWorkerNodes(workers int) ([]byte, error) { + cluster := &kindapi.Cluster{} + err := yaml.Unmarshal([]byte(kindConfig), cluster) + if err != nil { + return nil, err + } + cluster.Nodes = []kindapi.Node{ + {Role: kindapi.ControlPlaneRole}, + } + for i := 0; i < workers; i++ { + cluster.Nodes = append(cluster.Nodes, kindapi.Node{Role: kindapi.WorkerRole}) + } + return yaml.Marshal(cluster) +} + +func (k *KindManager) GetClient() (*client.Client, error) { + if k.client != nil { + return k.client, nil + } + + config, err := clientcmd.BuildConfigFromFlags("", k.KubeconfigLocation.Name()) + if err != nil { + return nil, err + } + + kclient, err := client.New(config, client.Options{}) + k.client = &kclient + if err != nil { + return nil, err + } + + return k.client, nil +} diff --git a/internal/envtest/multi_envtest.go b/internal/envtest/multi_envtest.go index e8da462..645de10 100644 --- a/internal/envtest/multi_envtest.go +++ b/internal/envtest/multi_envtest.go @@ -1,6 +1,9 @@ package envtest import ( + "os" + "strconv" + "sync" "testing" ctrl "sigs.k8s.io/controller-runtime" @@ -24,3 +27,40 @@ func RunMulti(m *testing.M, setupFunc func(e *MultiK8sEnvironment), numClusters exitCode := m.Run() return exitCode } + +func RunMultiKind(m *testing.M, setupFunc func(e *MultiK8sEnvironment), topology []int) (code int) { + e := make(MultiK8sEnvironment, len(topology)) + ctx := ctrl.SetupSignalHandler() + var wg sync.WaitGroup + for i := 0; i < len(topology); i++ { + cluster := KindManager{ + ClusterName: "cluster" + strconv.Itoa(i), + KubeconfigLocation: os.NewFile(0, GetBuildDir()+"/cluster"+strconv.Itoa(i)), + Nodes: topology[i], + } + wg.Add(1) + go func(i int) { + defer wg.Done() + e[i] = NewKindEnvironment(ctx, cluster) + e[i].Start() + }(i) + } + wg.Wait() + defer func() { + for i := 0; i < len(topology); i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + if err := e[i].KindCluster.TearDownKindCluster(); err != nil { + panic(err) + } + e[i].Stop() + }(i) + } + wg.Wait() + }() + + setupFunc(&e) + exitCode := m.Run() + return exitCode +} diff --git a/internal/envtest/root_dir.go b/internal/envtest/root_dir.go index 320fa88..02c5687 100644 --- a/internal/envtest/root_dir.go +++ b/internal/envtest/root_dir.go @@ -1,6 +1,7 @@ package envtest import ( + "os" "path/filepath" "runtime" ) @@ -10,3 +11,12 @@ func RootDir() string { _, b, _, _ := runtime.Caller(0) return filepath.Join(filepath.Dir(b), "../..") } + +func GetBuildDir() string { + buildDir := os.Getenv("BUILD_DIR") + if buildDir == "" { + _, b, _, _ := runtime.Caller(0) + buildDir = filepath.Join(filepath.Dir(b), "../..") + } + return buildDir +} From 5c3e445709e301b3e614d70aaa4a1827934c1e79 Mon Sep 17 00:00:00 2001 From: Miles Garnsey Date: Mon, 27 May 2024 17:50:36 +1000 Subject: [PATCH 2/3] Fix issues due to rebase. --- .../register/register_test.go | 69 +++++-------------- cmd/kubectl-k8ssandra/register/suite_test.go | 8 ++- internal/envtest/envtest.go | 3 +- internal/envtest/multi_envtest.go | 4 +- 4 files changed, 27 insertions(+), 57 deletions(-) diff --git a/cmd/kubectl-k8ssandra/register/register_test.go b/cmd/kubectl-k8ssandra/register/register_test.go index 16cb672..0931d93 100644 --- a/cmd/kubectl-k8ssandra/register/register_test.go +++ b/cmd/kubectl-k8ssandra/register/register_test.go @@ -2,8 +2,6 @@ package register import ( "context" - "errors" - "fmt" "os" "testing" "time" @@ -13,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,21 +39,15 @@ func TestRegister(t *testing.T) { } } return err1 == nil && err2 == nil - }, time.Second*60, time.Second*5) + }, time.Second*60, time.Second*2) - testDir, err := os.MkdirTemp("", "k8ssandra-operator-test-****") - require.NoError(err) - t.Cleanup(func() { - require.NoError(os.RemoveAll(testDir)) - }) - - kc1, err := (*multiEnv)[0].GetKubeconfig() - require.NoError(err) f1, err := os.Create(testDir + "/kubeconfig1") require.NoError(err) t.Cleanup(func() { require.NoError(f1.Close()) }) + kc1, err := (*multiEnv)[0].GetKubeconfig() + require.NoError(err) _, err = f1.Write(kc1) require.NoError(err) @@ -80,31 +73,18 @@ func TestRegister(t *testing.T) { Context: ctx, DestinationName: "test-destination", } - ctx := context.Background() - + // Continue reconciliation require.Eventually(func() bool { - if err := ex.RegisterCluster(); err != nil { - if errors.Is(err, NonRecoverableError{}) { - require.FailNow(fmt.Sprintf("Registration failed: %s", err.Error())) - } - if err.Error() == "no secret found for service account k8ssandra-operator" { - return true - } - return false - } - return true - }, time.Second*5, time.Millisecond*100) + res := ex.RegisterCluster() + return res == nil + }, time.Second*300, time.Second*1) - // This relies on a controller that is not running in the envtest. - - desiredSaSecret := &corev1.Secret{} - require.NoError(client1.Get(context.Background(), client.ObjectKey{Name: "k8ssandra-operator-secret", Namespace: "source-namespace"}, desiredSaSecret)) - patch := client.MergeFrom(desiredSaSecret.DeepCopy()) - desiredSaSecret.Data = map[string][]byte{ - "token": []byte("test-token"), - "ca.crt": []byte("test-ca"), - } - require.NoError(client1.Patch(ctx, desiredSaSecret, patch)) + sourceSecret := &corev1.Secret{} + // Ensure secret created. + require.Eventually(func() bool { + err := client1.Get(ctx, types.NamespacedName{Name: "k8ssandra-operator-secret", Namespace: "source-namespace"}, sourceSecret) + return err == nil + }, time.Second*60, time.Second*5) desiredSa := &corev1.ServiceAccount{} require.NoError(client1.Get( @@ -112,20 +92,6 @@ func TestRegister(t *testing.T) { client.ObjectKey{Name: "k8ssandra-operator", Namespace: "source-namespace"}, desiredSa)) - patch = client.MergeFrom(desiredSa.DeepCopy()) - desiredSa.Secrets = []corev1.ObjectReference{ - { - Name: "k8ssandra-operator-secret", - }, - } - require.NoError(client1.Patch(ctx, desiredSa, patch)) - - // Continue reconciliation - - require.Eventually(func() bool { - return ex.RegisterCluster() == nil - }, time.Second*3, time.Millisecond*100) - if err := configapi.AddToScheme(client2.Scheme()); err != nil { require.NoError(err) } @@ -145,15 +111,14 @@ func TestRegister(t *testing.T) { return false } return err == nil - }, time.Second*6, time.Millisecond*100) - + }, time.Second*60, time.Second*2) destKubeconfig := ClientConfigFromSecret(destSecret) require.Equal( - desiredSaSecret.Data["ca.crt"], + sourceSecret.Data["ca.crt"], destKubeconfig.Clusters["test-destination"].CertificateAuthorityData) require.Equal( - string(desiredSaSecret.Data["token"]), + string(sourceSecret.Data["token"]), destKubeconfig.AuthInfos["test-destination"].Token) } diff --git a/cmd/kubectl-k8ssandra/register/suite_test.go b/cmd/kubectl-k8ssandra/register/suite_test.go index 90491be..586bfa9 100644 --- a/cmd/kubectl-k8ssandra/register/suite_test.go +++ b/cmd/kubectl-k8ssandra/register/suite_test.go @@ -9,10 +9,16 @@ import ( var ( multiEnv *envtest.MultiK8sEnvironment + testDir string + err error ) func TestMain(m *testing.M) { + testDir, err = os.MkdirTemp("", "k8ssandra-operator-test") + if err != nil { + panic(err.Error()) + } os.Exit(envtest.RunMultiKind(m, func(e *envtest.MultiK8sEnvironment) { multiEnv = e - }, []int{1, 1})) + }, []int{1, 1}, testDir)) } diff --git a/internal/envtest/envtest.go b/internal/envtest/envtest.go index dde3a5b..f9ea24e 100644 --- a/internal/envtest/envtest.go +++ b/internal/envtest/envtest.go @@ -72,14 +72,13 @@ func NewKindEnvironment(ctx context.Context, cluster KindManager) *Environment { ctx, cancel := context.WithCancel(ctx) env.Context = ctx env.cancelManager = cancel - env.Client = *cluster.client env.Kubeconfig = cluster.KubeconfigLocation.Name() env.env.Config = cluster.RestConfig env.KindCluster = cluster return env } -func (e *Environment) GetClient(namespace string) client.Client { +func (e *Environment) GetClientInNamespace(namespace string) client.Client { c, err := kubernetes.GetClientInNamespace(e.env.Config, namespace) if err != nil { panic(err) diff --git a/internal/envtest/multi_envtest.go b/internal/envtest/multi_envtest.go index 645de10..f5ad281 100644 --- a/internal/envtest/multi_envtest.go +++ b/internal/envtest/multi_envtest.go @@ -28,14 +28,14 @@ func RunMulti(m *testing.M, setupFunc func(e *MultiK8sEnvironment), numClusters return exitCode } -func RunMultiKind(m *testing.M, setupFunc func(e *MultiK8sEnvironment), topology []int) (code int) { +func RunMultiKind(m *testing.M, setupFunc func(e *MultiK8sEnvironment), topology []int, testDir string) (code int) { e := make(MultiK8sEnvironment, len(topology)) ctx := ctrl.SetupSignalHandler() var wg sync.WaitGroup for i := 0; i < len(topology); i++ { cluster := KindManager{ ClusterName: "cluster" + strconv.Itoa(i), - KubeconfigLocation: os.NewFile(0, GetBuildDir()+"/cluster"+strconv.Itoa(i)), + KubeconfigLocation: os.NewFile(0, testDir+"/cluster"+strconv.Itoa(i)), Nodes: topology[i], } wg.Add(1) From f714b1e288a555f91f3572aadc1627d12ab42167 Mon Sep 17 00:00:00 2001 From: Miles Garnsey Date: Wed, 29 May 2024 11:19:39 +1000 Subject: [PATCH 3/3] Micke's feedback. --- cmd/kubectl-k8ssandra/register/register_test.go | 9 +++++---- cmd/kubectl-k8ssandra/register/suite_test.go | 2 +- internal/envtest/root_dir.go | 10 ---------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/cmd/kubectl-k8ssandra/register/register_test.go b/cmd/kubectl-k8ssandra/register/register_test.go index 0931d93..458492e 100644 --- a/cmd/kubectl-k8ssandra/register/register_test.go +++ b/cmd/kubectl-k8ssandra/register/register_test.go @@ -39,7 +39,7 @@ func TestRegister(t *testing.T) { } } return err1 == nil && err2 == nil - }, time.Second*60, time.Second*2) + }, time.Second*6, time.Millisecond*100) f1, err := os.Create(testDir + "/kubeconfig1") require.NoError(err) @@ -77,14 +77,14 @@ func TestRegister(t *testing.T) { require.Eventually(func() bool { res := ex.RegisterCluster() return res == nil - }, time.Second*300, time.Second*1) + }, time.Second*6, time.Millisecond*100) sourceSecret := &corev1.Secret{} // Ensure secret created. require.Eventually(func() bool { err := client1.Get(ctx, types.NamespacedName{Name: "k8ssandra-operator-secret", Namespace: "source-namespace"}, sourceSecret) return err == nil - }, time.Second*60, time.Second*5) + }, time.Second*6, time.Millisecond*100) desiredSa := &corev1.ServiceAccount{} require.NoError(client1.Get( @@ -111,7 +111,8 @@ func TestRegister(t *testing.T) { return false } return err == nil - }, time.Second*60, time.Second*2) + }, time.Second*6, time.Millisecond*100) + destKubeconfig := ClientConfigFromSecret(destSecret) require.Equal( sourceSecret.Data["ca.crt"], diff --git a/cmd/kubectl-k8ssandra/register/suite_test.go b/cmd/kubectl-k8ssandra/register/suite_test.go index 586bfa9..062f9a6 100644 --- a/cmd/kubectl-k8ssandra/register/suite_test.go +++ b/cmd/kubectl-k8ssandra/register/suite_test.go @@ -14,7 +14,7 @@ var ( ) func TestMain(m *testing.M) { - testDir, err = os.MkdirTemp("", "k8ssandra-operator-test") + testDir, err = os.MkdirTemp("", "k8ssandra-client-test") if err != nil { panic(err.Error()) } diff --git a/internal/envtest/root_dir.go b/internal/envtest/root_dir.go index 02c5687..320fa88 100644 --- a/internal/envtest/root_dir.go +++ b/internal/envtest/root_dir.go @@ -1,7 +1,6 @@ package envtest import ( - "os" "path/filepath" "runtime" ) @@ -11,12 +10,3 @@ func RootDir() string { _, b, _, _ := runtime.Caller(0) return filepath.Join(filepath.Dir(b), "../..") } - -func GetBuildDir() string { - buildDir := os.Getenv("BUILD_DIR") - if buildDir == "" { - _, b, _, _ := runtime.Caller(0) - buildDir = filepath.Join(filepath.Dir(b), "../..") - } - return buildDir -}