From e3d05202b5273522696a9f603c971b4b1e3f342b Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 16 Dec 2024 13:05:56 +0100 Subject: [PATCH] WIP: Evaluating CreateOrUpdate Signed-off-by: Danil-Grigorev --- .../chart-upgrade/chart_upgrade_test.go | 3 +- .../e2e/suites/import-gitops-v3/suite_test.go | 3 +- test/e2e/suites/import-gitops/suite_test.go | 3 +- test/e2e/suites/migrate-gitops/suite_test.go | 3 +- .../update-labels/update_labels_test.go | 4 +- test/e2e/suites/v2prov/v2prov_test.go | 4 +- test/framework/apply.go | 76 +++++++++++++++++++ test/framework/apply_template_helper.go | 2 +- test/framework/chartmuseum_helper.go | 4 +- test/framework/fleet_helper.go | 2 +- test/testenv/gitea.go | 2 +- test/testenv/operator.go | 2 +- test/testenv/rancher.go | 8 +- test/testenv/turtles.go | 4 +- 14 files changed, 100 insertions(+), 20 deletions(-) create mode 100644 test/framework/apply.go diff --git a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go index 8c03e679..6c6f3c68 100644 --- a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go +++ b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go @@ -33,6 +33,7 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + turtlesframework "github.com/rancher/turtles/test/framework" capiframework "sigs.k8s.io/cluster-api/test/framework" ) @@ -96,7 +97,7 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL }, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) By("Setting the CAAPF config to use hostNetwork") - Expect(bootstrapClusterProxy.Apply(ctx, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) + Expect(turtlesframework.Apply(ctx, bootstrapClusterProxy, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) }) upgradeInput.PostUpgradeSteps = append(upgradeInput.PostUpgradeSteps, func() { diff --git a/test/e2e/suites/import-gitops-v3/suite_test.go b/test/e2e/suites/import-gitops-v3/suite_test.go index 378c58c6..72324170 100644 --- a/test/e2e/suites/import-gitops-v3/suite_test.go +++ b/test/e2e/suites/import-gitops-v3/suite_test.go @@ -35,6 +35,7 @@ import ( "github.com/rancher/turtles/test/e2e" "github.com/rancher/turtles/test/framework" + turtlesframework "github.com/rancher/turtles/test/framework" "github.com/rancher/turtles/test/testenv" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -187,7 +188,7 @@ var _ = SynchronizedBeforeSuite( }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) By("Setting the CAAPF config to use hostNetwork") - Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) + Expect(turtlesframework.Apply(ctx, setupClusterResult.BootstrapClusterProxy, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) if !shortTestOnly() && !localTestOnly() { By("Running full tests, deploying additional infrastructure providers") diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index ff6915a8..e676c147 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -34,6 +34,7 @@ import ( . "github.com/onsi/gomega" "github.com/rancher/turtles/test/e2e" "github.com/rancher/turtles/test/framework" + turtlesframework "github.com/rancher/turtles/test/framework" "github.com/rancher/turtles/test/testenv" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -201,7 +202,7 @@ var _ = SynchronizedBeforeSuite( }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) By("Setting the CAAPF config to use hostNetwork") - Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) + Expect(turtlesframework.Apply(ctx, setupClusterResult.BootstrapClusterProxy, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) } else { rtInput := testenv.DeployRancherTurtlesInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, diff --git a/test/e2e/suites/migrate-gitops/suite_test.go b/test/e2e/suites/migrate-gitops/suite_test.go index e3b92c29..42b582bf 100644 --- a/test/e2e/suites/migrate-gitops/suite_test.go +++ b/test/e2e/suites/migrate-gitops/suite_test.go @@ -37,6 +37,7 @@ import ( "github.com/rancher/turtles/test/testenv" appsv1 "k8s.io/api/apps/v1" + turtlesframework "github.com/rancher/turtles/test/framework" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" capiframework "sigs.k8s.io/cluster-api/test/framework" @@ -202,7 +203,7 @@ var _ = SynchronizedBeforeSuite( }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) By("Setting the CAAPF config to use hostNetwork") - Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) + Expect(turtlesframework.Apply(ctx, setupClusterResult.BootstrapClusterProxy, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) giteaInput := testenv.DeployGiteaInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, diff --git a/test/e2e/suites/update-labels/update_labels_test.go b/test/e2e/suites/update-labels/update_labels_test.go index 9a927bc3..d89e17a5 100644 --- a/test/e2e/suites/update-labels/update_labels_test.go +++ b/test/e2e/suites/update-labels/update_labels_test.go @@ -108,7 +108,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w } }) Expect(err).ToNot(HaveOccurred()) - Expect(bootstrapClusterProxy.Apply(ctx, []byte(rkeConfig))).To(Succeed(), "Failed apply Digital Ocean RKE config") + Expect(turtlesframework.Apply(ctx, bootstrapClusterProxy, []byte(rkeConfig))).To(Succeed(), "Failed apply Digital Ocean RKE config") cluster, err := envsubst.Eval(string(e2e.V2ProvAzureCluster), func(s string) string { switch s { @@ -127,7 +127,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w } }) Expect(err).ToNot(HaveOccurred()) - Expect(bootstrapClusterProxy.Apply(ctx, []byte(cluster))).To(Succeed(), "Failed apply Digital Ocean cluster config") + Expect(turtlesframework.Apply(ctx, bootstrapClusterProxy, []byte(cluster))).To(Succeed(), "Failed apply Digital Ocean cluster config") By("Waiting for the rancher cluster record to appear") rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/suites/v2prov/v2prov_test.go b/test/e2e/suites/v2prov/v2prov_test.go index add1603a..b4911666 100644 --- a/test/e2e/suites/v2prov/v2prov_test.go +++ b/test/e2e/suites/v2prov/v2prov_test.go @@ -109,7 +109,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w } }) Expect(err).ToNot(HaveOccurred()) - Expect(bootstrapClusterProxy.Apply(ctx, []byte(rkeConfig))).To(Succeed(), "Failed apply Digital Ocean RKE config") + Expect(turtlesframework.Apply(ctx, bootstrapClusterProxy, []byte(rkeConfig))).To(Succeed(), "Failed apply Digital Ocean RKE config") cluster, err := envsubst.Eval(string(e2e.V2ProvAzureCluster), func(s string) string { switch s { @@ -128,7 +128,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w } }) Expect(err).ToNot(HaveOccurred()) - Expect(bootstrapClusterProxy.Apply(ctx, []byte(cluster))).To(Succeed(), "Failed apply Digital Ocean cluster config") + Expect(turtlesframework.Apply(ctx, bootstrapClusterProxy, []byte(cluster))).To(Succeed(), "Failed apply Digital Ocean cluster config") By("Waiting for the rancher cluster record to appear") rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{ diff --git a/test/framework/apply.go b/test/framework/apply.go new file mode 100644 index 00000000..e54d0ea7 --- /dev/null +++ b/test/framework/apply.go @@ -0,0 +1,76 @@ +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + + . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api/test/framework" + capiexec "sigs.k8s.io/cluster-api/test/framework/exec" +) + +// Apply wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster. +func Apply(ctx context.Context, p framework.ClusterProxy, resources []byte, args ...string) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") + Expect(resources).NotTo(BeNil(), "resources is required for Apply") + + if err := KubectlApply(ctx, p.GetKubeconfigPath(), resources, args...); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return errors.New(fmt.Sprintf("%s: stderr: %s", err.Error(), exitErr.Stderr)) + } + return err + } + + return nil +} + +// KubectlApply shells out to kubectl apply. +// +// TODO: Remove this usage of kubectl and replace with a function from apply.go using the controller-runtime client. +func KubectlApply(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error { + aargs := append([]string{"apply", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...) + rbytes := bytes.NewReader(resources) + applyCmd := capiexec.NewCommand( + capiexec.WithCommand(kubectlPath()), + capiexec.WithArgs(aargs...), + capiexec.WithStdin(rbytes), + ) + fmt.Printf("Running kubectl %s\n", strings.Join(aargs, " ")) + stdout, stderr, err := applyCmd.Run(ctx) + if len(stderr) > 0 { + fmt.Printf("stderr:\n%s\n", string(stderr)) + } + if len(stdout) > 0 { + fmt.Printf("stdout:\n%s\n", string(stdout)) + } + return err +} + +func kubectlPath() string { + if kubectlPath, ok := os.LookupEnv("CAPI_KUBECTL_PATH"); ok { + return kubectlPath + } + return "kubectl" +} diff --git a/test/framework/apply_template_helper.go b/test/framework/apply_template_helper.go index f5ec54a3..33055903 100644 --- a/test/framework/apply_template_helper.go +++ b/test/framework/apply_template_helper.go @@ -76,5 +76,5 @@ func ApplyFromTemplate(ctx context.Context, input ApplyFromTemplateInput) error return os.WriteFile(input.OutputFilePath, []byte(template), os.ModePerm) } - return input.Proxy.Apply(ctx, []byte(template)) + return Apply(ctx, input.Proxy, []byte(template)) } diff --git a/test/framework/chartmuseum_helper.go b/test/framework/chartmuseum_helper.go index 3d3395cf..3286c08d 100644 --- a/test/framework/chartmuseum_helper.go +++ b/test/framework/chartmuseum_helper.go @@ -88,7 +88,7 @@ func DeployChartMuseum(ctx context.Context, input ChartMuseumInput) string { ).CombinedOutput() By("Creating chartmuseum manifests") - Expect(input.Proxy.Apply(ctx, input.ChartMuseumManifests)).ShouldNot(HaveOccurred()) + Expect(Apply(ctx, input.Proxy, input.ChartMuseumManifests)).ShouldNot(HaveOccurred()) By("Waiting for chartmuseum rollout") framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ @@ -117,7 +117,7 @@ func DeployChartMuseum(ctx context.Context, input ChartMuseumInput) string { variableGetter := GetVariable(input.Variables) ingress, err := envsubst.Eval(string(input.CustomIngressConfig), variableGetter) Expect(err).ToNot(HaveOccurred()) - Expect(input.Proxy.Apply(ctx, []byte(ingress))).To(Succeed()) + Expect(Apply(ctx, input.Proxy, []byte(ingress))).To(Succeed()) By("Getting git server ingress address") host := GetIngressHost(ctx, GetIngressHostInput{ diff --git a/test/framework/fleet_helper.go b/test/framework/fleet_helper.go index 23b4cfe8..bda73e7a 100644 --- a/test/framework/fleet_helper.go +++ b/test/framework/fleet_helper.go @@ -88,7 +88,7 @@ func FleetCreateGitRepo(ctx context.Context, input FleetCreateGitRepoInput) { By("Applying GitRepo") Eventually(func() error { - return input.ClusterProxy.Apply(ctx, renderedTemplate.Bytes()) + return Apply(ctx, input.ClusterProxy, renderedTemplate.Bytes()) }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to appl GitRepo") } diff --git a/test/testenv/gitea.go b/test/testenv/gitea.go index 4ffab218..e722b84d 100644 --- a/test/testenv/gitea.go +++ b/test/testenv/gitea.go @@ -207,7 +207,7 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult variableGetter := turtlesframework.GetVariable(input.Variables) ingress, err := envsubst.Eval(string(input.CustomIngressConfig), variableGetter) Expect(err).ToNot(HaveOccurred()) - Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(ingress))).To(Succeed()) + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, []byte(ingress))).To(Succeed()) By("Getting git server ingress address") host := turtlesframework.GetIngressHost(ctx, turtlesframework.GetIngressHostInput{ diff --git a/test/testenv/operator.go b/test/testenv/operator.go index 6e398304..7512b684 100644 --- a/test/testenv/operator.go +++ b/test/testenv/operator.go @@ -83,7 +83,7 @@ func CAPIOperatorDeployProvider(ctx context.Context, input CAPIOperatorDeployPro } By("Adding CAPI Operator providers") - Expect(input.BootstrapClusterProxy.Apply(ctx, input.CAPIProvidersYAML)).To(Succeed(), "Failed to add CAPI operator providers") + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.CAPIProvidersYAML)).To(Succeed(), "Failed to add CAPI operator providers") if len(input.WaitForDeployments) == 0 { By("No deployments to wait for") diff --git a/test/testenv/rancher.go b/test/testenv/rancher.go index 8cbc546f..03022232 100644 --- a/test/testenv/rancher.go +++ b/test/testenv/rancher.go @@ -278,11 +278,11 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) { By("Setting up ingress") ingress, err := envsubst.Eval(string(input.RancherIngressConfig), os.Getenv) Expect(err).ToNot(HaveOccurred()) - Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(ingress))).To(Succeed()) + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, []byte(ingress))).To(Succeed()) } if len(input.RancherServicePatch) > 0 { By("Updating rancher svc") - Expect(input.BootstrapClusterProxy.Apply(ctx, input.RancherServicePatch, "--server-side")).To(Succeed()) + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.RancherServicePatch)).To(Succeed()) } By("Waiting for rancher webhook rollout") @@ -432,7 +432,7 @@ func RancherDeployIngress(ctx context.Context, input RancherDeployIngressInput) func deployIsolatedModeIngress(ctx context.Context, input RancherDeployIngressInput) { By("Deploying custom ingress") - Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(input.CustomIngress))).To(Succeed()) + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, []byte(input.CustomIngress))).To(Succeed()) By("Getting custom ingress deployment") ingressDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: input.CustomIngressDeployment, Namespace: input.CustomIngressNamespace}} @@ -525,7 +525,7 @@ func deployNgrokIngress(ctx context.Context, input RancherDeployIngressInput) { Expect(err).ToNot(HaveOccurred()) By("Setting up default ingress class") - Expect(input.BootstrapClusterProxy.Apply(ctx, input.DefaultIngressClassPatch, "--server-side")).To(Succeed()) + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.DefaultIngressClassPatch)).To(Succeed()) } // PreRancherInstallHookInput represents the input parameters for the pre-Rancher install hook. diff --git a/test/testenv/turtles.go b/test/testenv/turtles.go index cd1918d2..f20787c7 100644 --- a/test/testenv/turtles.go +++ b/test/testenv/turtles.go @@ -114,7 +114,7 @@ func DeployRancherTurtles(ctx context.Context, input DeployRancherTurtlesInput) if input.CAPIProvidersSecretYAML != nil { By("Adding CAPI variables secret") - Expect(input.BootstrapClusterProxy.Apply(ctx, input.CAPIProvidersSecretYAML)).To(Succeed()) + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.CAPIProvidersSecretYAML)).To(Succeed()) } chartPath := input.TurtlesChartPath @@ -185,7 +185,7 @@ func DeployRancherTurtles(ctx context.Context, input DeployRancherTurtlesInput) // TODO: this can probably be covered by the Operator helper By("Adding CAPI infrastructure providers") - Expect(input.BootstrapClusterProxy.Apply(ctx, input.CAPIProvidersYAML)).To(Succeed()) + Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.CAPIProvidersYAML)).To(Succeed()) By("Waiting for CAPI deployment to be available") framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{