From c3fa2ab8e13fb7de1ad09a1b8314c80631fcf57b Mon Sep 17 00:00:00 2001 From: Dheeraj Singh Jodha Date: Mon, 18 Dec 2023 14:28:44 +0530 Subject: [PATCH 1/2] fix(STONEINTG-704): add checks for finalizer removal from Intg PLRs After this commit, we are now checking if the finalizer gets removed succesfully from all the Integration PLRs related to the given Application, after the integration has marked the snapshot as passed/failed AND reported the test results back to the status annotation of the snapshot. Signed-off-by: Dheeraj --- pkg/clients/integration/pipelineruns.go | 40 ++++++++++++++++++++++++ tests/integration-service/integration.go | 30 +++++++++++------- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/pkg/clients/integration/pipelineruns.go b/pkg/clients/integration/pipelineruns.go index e1adf4c52..1f84d937d 100644 --- a/pkg/clients/integration/pipelineruns.go +++ b/pkg/clients/integration/pipelineruns.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "knative.dev/pkg/apis" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // CreateIntegrationPipelineRun creates new integrationPipelineRun. @@ -180,6 +181,45 @@ func (i *IntegrationController) WaitForAllIntegrationPipelinesToBeFinished(testN return nil } +// WaitForFinalizerToGetRemovedFromAllIntegrationPipelineRuns waits for +// the given finalizer to get removed from all integration pipelinesruns +// that are related to the given application and namespace. +func (i *IntegrationController) WaitForFinalizerToGetRemovedFromAllIntegrationPipelineRuns(testNamespace, applicationName string, snapshot *appstudioApi.Snapshot) error { + integrationTestScenarios, err := i.GetIntegrationTestScenarios(applicationName, testNamespace) + if err != nil { + return fmt.Errorf("unable to get IntegrationTestScenarios for Application %s/%s. Error: %v", testNamespace, applicationName, err) + } + + for _, testScenario := range *integrationTestScenarios { + testScenario := testScenario + GinkgoWriter.Printf("Integration test scenario %s is found\n", testScenario.Name) + err = i.WaitForFinalizerToGetRemovedFromIntegrationPipeline(&testScenario, snapshot, testNamespace) + if err != nil { + return fmt.Errorf("error occurred while waiting for Integration PLR (associated with IntegrationTestScenario: %s) to NOT have the finalizer. Error: %v", testScenario.Name, err) + } + } + + return nil +} + +// WaitForFinalizerToGetRemovedFromIntegrationPipeline waits for the +// given finalizer to get removed from the given integration pipelinerun +func (i *IntegrationController) WaitForFinalizerToGetRemovedFromIntegrationPipeline(testScenario *integrationv1beta1.IntegrationTestScenario, snapshot *appstudioApi.Snapshot, appNamespace string) error { + return wait.PollUntilContextTimeout(context.Background(), constants.PipelineRunPollingInterval, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + pipelineRun, err := i.GetIntegrationPipelineRun(testScenario.Name, snapshot.Name, appNamespace) + if err != nil { + GinkgoWriter.Println("PipelineRun has not been created yet for test scenario %s and snapshot %s/%s", testScenario.GetName(), snapshot.GetNamespace(), snapshot.GetName()) + return false, nil + } + if controllerutil.ContainsFinalizer(pipelineRun, "test.appstudio.openshift.io/pipelinerun") { + GinkgoWriter.Printf("build pipelineRun %s/%s still contains the finalizer: %s", pipelineRun.GetNamespace(), pipelineRun.GetName(), "test.appstudio.openshift.io/pipelinerun") + return false, nil + } + + return true, nil + }) +} + // GetAnnotationIfExists returns the value of a given annotation within a pipelinerun, if it exists. func (i *IntegrationController) GetAnnotationIfExists(testNamespace, applicationName, componentName, annotationKey string) (string, error) { pipelineRun, err := i.GetBuildPipelineRun(componentName, applicationName, testNamespace, false, "") diff --git a/tests/integration-service/integration.go b/tests/integration-service/integration.go index 1eefd5b5b..bc05abe1d 100644 --- a/tests/integration-service/integration.go +++ b/tests/integration-service/integration.go @@ -100,6 +100,19 @@ var _ = framework.IntegrationServiceSuiteDescribe("Integration Service E2E tests Expect(f.AsKubeDeveloper.IntegrationController.WaitForBuildPipelineRunToGetAnnotated(testNamespace, applicationName, componentName, snapshotAnnotation)).To(Succeed()) }) + It("verifies that the finalizer has been removed from the build pipelinerun", func() { + timeout := "60s" + interval := "1s" + Eventually(func() error { + pipelineRun, err = f.AsKubeDeveloper.IntegrationController.GetBuildPipelineRun(componentName, applicationName, testNamespace, false, "") + Expect(err).ShouldNot(HaveOccurred()) + if controllerutil.ContainsFinalizer(pipelineRun, pipelinerunFinalizerByIntegrationService) { + return fmt.Errorf("build pipelineRun %s/%s still contains the finalizer: %s", pipelineRun.GetNamespace(), pipelineRun.GetName(), pipelinerunFinalizerByIntegrationService) + } + return nil + }, timeout, interval).Should(Succeed(), "timeout when waiting for finalizer to be removed") + }) + It("checks if all of the integrationPipelineRuns passed", Label("slow"), func() { Expect(f.AsKubeDeveloper.IntegrationController.WaitForAllIntegrationPipelinesToBeFinished(testNamespace, applicationName, snapshot)).To(Succeed()) }) @@ -121,17 +134,8 @@ var _ = framework.IntegrationServiceSuiteDescribe("Integration Service E2E tests }, timeout, interval).Should(Succeed()) }) - It("verifies that the finalizer has been removed from the build pipelinerun", func() { - timeout := "60s" - interval := "1s" - Eventually(func() error { - pipelineRun, err = f.AsKubeDeveloper.IntegrationController.GetBuildPipelineRun(componentName, applicationName, testNamespace, false, "") - Expect(err).ShouldNot(HaveOccurred()) - if controllerutil.ContainsFinalizer(pipelineRun, pipelinerunFinalizerByIntegrationService) { - return fmt.Errorf("build pipelineRun %s/%s still contains the finalizer: %s", pipelineRun.GetNamespace(), pipelineRun.GetName(), pipelinerunFinalizerByIntegrationService) - } - return nil - }, timeout, interval).Should(Succeed(), "timeout when waiting for finalizer to be removed") + It("checks if the finalizer was removed from all of the related Integration pipelineRuns", func() { + Expect(f.AsKubeDeveloper.IntegrationController.WaitForFinalizerToGetRemovedFromAllIntegrationPipelineRuns(testNamespace, applicationName, snapshot)).To(Succeed()) }) }) @@ -265,6 +269,10 @@ var _ = framework.IntegrationServiceSuiteDescribe("Integration Service E2E tests Expect(f.AsKubeAdmin.CommonController.HaveTestsSucceeded(snapshot)).To(BeFalse(), "expected tests to fail for snapshot %s/%s", snapshot.GetNamespace(), snapshot.GetName()) }) + It("checks if the finalizer was removed from all of the related Integration pipelineRuns", func() { + Expect(f.AsKubeDeveloper.IntegrationController.WaitForFinalizerToGetRemovedFromAllIntegrationPipelineRuns(testNamespace, applicationName, snapshot)).To(Succeed()) + }) + It("creates a new IntegrationTestScenario", func() { newIntegrationTestScenario, err = f.AsKubeAdmin.IntegrationController.CreateIntegrationTestScenario(applicationName, testNamespace, BundleURL, InPipelineName) Expect(err).ShouldNot(HaveOccurred()) From 4a4d94a46bae59df0addcc2767d9890bc293368b Mon Sep 17 00:00:00 2001 From: Dheeraj Singh Jodha Date: Tue, 2 Jan 2024 15:53:19 +0530 Subject: [PATCH 2/2] fix(STONEINTG-704): update the Readme.md file with test suite details * and also update the GoDoc for a function Signed-off-by: Dheeraj --- pkg/clients/integration/pipelineruns.go | 3 ++- tests/integration-service/README.md | 15 +++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/clients/integration/pipelineruns.go b/pkg/clients/integration/pipelineruns.go index 1f84d937d..b00c61361 100644 --- a/pkg/clients/integration/pipelineruns.go +++ b/pkg/clients/integration/pipelineruns.go @@ -91,7 +91,8 @@ func (i *IntegrationController) GetBuildPipelineRun(componentName, applicationNa return pipelineRun, err } -// GetComponentPipeline returns the pipeline for a given component labels. +// GetIntegrationPipelineRun returns the integration pipelineRun +// for a given scenario, snapshot labels. func (i *IntegrationController) GetIntegrationPipelineRun(integrationTestScenarioName string, snapshotName string, namespace string) (*tektonv1beta1.PipelineRun, error) { opts := []client.ListOption{ client.InNamespace(namespace), diff --git a/tests/integration-service/README.md b/tests/integration-service/README.md index e1b3232e0..ff22394ca 100644 --- a/tests/integration-service/README.md +++ b/tests/integration-service/README.md @@ -10,13 +10,15 @@ Happy path testing describes tests that focus on the most common scenarios while - Testing for successful creation of applications and components. - Asserting the successful creation of a snapshot after a push event. -- Checking if the BuildPipelineRun is successfully triggered and completed. +- Checking if the BuildPipelineRun is successfully triggered, contains the finalizer, and got completed. - Asserting the signing of BuildPipelineRun. -- Validating the successful creation of a SnapshotEnvironmentBinding. -- Validating the successful creation of a Release. -- Validating the Global Candidate is updated +- Validating the successful creation of a Snapshot, and removal of finalizer from BuildPipelineRun. - Verifying that all the Integration PipelineRuns finished successfully. +- Checking that the status of integration tests were reported to the Snapshot. - Checking that a snapshot gets successfully marked as 'passed' after all Integration pipeline runs finished. +- Validating the Global Candidate is updated +- Validating the successful creation of a SnapshotEnvironmentBinding. +- Validating the successful creation of a Release. ### E2E tests of Namespace-backed Environments (within `namespace-backed-environments.go`): @@ -48,6 +50,11 @@ Happy path testing describes tests that focus on the most common scenarios while - Creating an IntegrationTestScenario that is expected to fail. - Asserting that a snapshot is marked as failed. +- Creating a new IntegrationTestScenario that is expected to pass. +- Updating the Snapshot with a re-run label for the new scenario. +- Validating that a new Integration PLR is created and finished. +- Asserting that the Snapshot doesn't contain re-run label, and contains the name of re-triggered pipelinerun. +- Asserting that a snapshot is still marked as failed. - Validating that no Release CRs and no SnapshotEnvironmentBinding are created in certain scenarios. - Checking that the global candidate does not get updated unexpectedly.