feat(STONEINTG-702): add tests for integration pipeline re-runs

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/redhat-appstudio/integration-service v0.0.0-20231017154915-ca19edc57d63
 	github.com/redhat-appstudio/jvm-build-service v0.0.0-20230821060312-4172397d68e8
 	github.com/redhat-appstudio/managed-gitops/backend-shared v0.0.0
+	github.com/redhat-appstudio/operator-toolkit v0.0.0-20230913085326-6c5e9d368a6a
 	github.com/redhat-appstudio/release-service v0.0.0-20231012135118-498f8de95cc4
 	github.com/redhat-appstudio/remote-secret v0.0.0-20230713072146-a6094c712436
 	github.com/redhat-appstudio/service-provider-integration-operator v0.2023.22-0.20230713080056-eae17aa8c172
@@ -260,7 +261,6 @@ require (
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/prometheus/statsd_exporter v0.23.1 // indirect
 	github.com/redhat-appstudio/application-service v0.0.0-20230717184417-67d31a01a776 // indirect
-	github.com/redhat-appstudio/operator-toolkit v0.0.0-20230913085326-6c5e9d368a6a // indirect
 	github.com/redhat-developer/alizer/go v0.0.0-20230516215932-135a2bb3fb90 // indirect
 	github.com/redhat-developer/gitops-generator v0.0.0-20230614175323-aff86c6bc55e // indirect
 	github.com/redis/go-redis/v9 v9.0.5 // indirect

pkg/clients/integration/snapshots.go

@@ -118,6 +118,13 @@ func (i *IntegrationController) DeleteSnapshot(hasSnapshot *appstudioApi.Snapsho
 	return err
 }
 
+// PatchSnapshot patches the given snapshot with the provided patch.
+func (i *IntegrationController) PatchSnapshot(oldSnapshot *appstudioApi.Snapshot, newSnapshot *appstudioApi.Snapshot) error {
+	patch := client.MergeFrom(oldSnapshot)
+	err := i.KubeRest().Patch(context.Background(), newSnapshot, patch)
+	return err
+}
+
 // DeleteAllSnapshotsInASpecificNamespace removes all snapshots from a specific namespace. Useful when creating a lot of resources and want to remove all of them
 func (i *IntegrationController) DeleteAllSnapshotsInASpecificNamespace(namespace string, timeout time.Duration) error {
 	if err := i.KubeRest().DeleteAllOf(context.Background(), &appstudioApi.Snapshot{}, client.InNamespace(namespace)); err != nil {

tests/integration-service/const.go

@@ -38,6 +38,7 @@ const (
 	snapshotAnnotation                       = "appstudio.openshift.io/snapshot"
 	scenarioAnnotation                       = "test.appstudio.openshift.io/scenario"
 	pipelinerunFinalizerByIntegrationService = "test.appstudio.openshift.io/pipelinerun"
+	snapshotRerunLabel                       = "test.appstudio.openshift.io/run"
 
 	chainsSignedAnnotation = "chains.tekton.dev/signed"
 )

tests/integration-service/integration.go

@@ -2,6 +2,7 @@ package integration
 
 import (
 	"fmt"
+	"github.com/redhat-appstudio/operator-toolkit/metadata"
 	"time"
 
 	"github.com/devfile/library/v2/pkg/util"
@@ -28,6 +29,7 @@ var _ = framework.IntegrationServiceSuiteDescribe("Integration Service E2E tests
 
 	var applicationName, componentName, testNamespace string
 	var integrationTestScenario *integrationv1alpha1.IntegrationTestScenario
+	var newIntegrationTestScenario *integrationv1alpha1.IntegrationTestScenario
 	var timeout, interval time.Duration
 	var originalComponent *appstudioApi.Component
 	var pipelineRun *v1beta1.PipelineRun
@@ -263,6 +265,69 @@ var _ = framework.IntegrationServiceSuiteDescribe("Integration Service E2E tests
 			Expect(f.AsKubeAdmin.CommonController.HaveTestsSucceeded(snapshot)).To(BeFalse(), "expected tests to fail for snapshot %s/%s", snapshot.GetNamespace(), snapshot.GetName())
 		})
 
+		It("creates a new IntegrationTestScenario", func() {
+			newIntegrationTestScenario, err = f.AsKubeAdmin.IntegrationController.CreateIntegrationTestScenario(applicationName, testNamespace, BundleURL, InPipelineName)
+			Expect(err).ShouldNot(HaveOccurred())
+		})
+
+		It("updates the Snapshot with the re-run label for the new scenario", FlakeAttempts(3), func() {
+			updatedSnapshot := snapshot.DeepCopy()
+			err := metadata.AddLabels(updatedSnapshot, map[string]string{snapshotRerunLabel: newIntegrationTestScenario.Name})
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(f.AsKubeAdmin.IntegrationController.PatchSnapshot(snapshot, updatedSnapshot)).Should(Succeed())
+			Expect(metadata.GetLabelsWithPrefix(updatedSnapshot, snapshotRerunLabel)).NotTo(BeEmpty())
+		})
+
+		When("An snapshot is updated with a re-run label for a given scenario", func() {
+			It("checks if the new integration pipelineRun started", Label("slow"), func() {
+				reRunPipelineRun, err := f.AsKubeDeveloper.IntegrationController.WaitForIntegrationPipelineToGetStarted(newIntegrationTestScenario.Name, snapshot.Name, testNamespace)
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(reRunPipelineRun).ShouldNot(BeNil())
+			})
+
+			It("checks if the re-run label was removed from the Snapshot", func() {
+				Eventually(func() error {
+					snapshot, err = f.AsKubeAdmin.IntegrationController.GetSnapshot(snapshot.Name, "", "", testNamespace)
+					if err != nil {
+						return fmt.Errorf("encountered error while getting Snapshot %s/%s: %w", snapshot.Name, snapshot.Namespace, err)
+					}
+
+					if metadata.HasLabel(snapshot, snapshotRerunLabel) {
+						return fmt.Errorf("the Snapshot %s/%s shouldn't contain the %s label", snapshot.Name, snapshot.Namespace, snapshotRerunLabel)
+					}
+					return nil
+				}, timeout, interval).Should(Succeed())
+			})
+
+			It("checks if all integration pipelineRuns finished successfully", Label("slow"), func() {
+				Expect(f.AsKubeDeveloper.IntegrationController.WaitForAllIntegrationPipelinesToBeFinished(testNamespace, applicationName, snapshot)).To(Succeed())
+			})
+
+			It("checks if the name of the re-triggered pipelinerun is reported in the Snapshot", FlakeAttempts(3), func() {
+				Eventually(func(g Gomega) {
+					snapshot, err = f.AsKubeAdmin.IntegrationController.GetSnapshot(snapshot.Name, "", "", testNamespace)
+					g.Expect(err).ShouldNot(HaveOccurred())
+
+					statusDetail, err := f.AsKubeDeveloper.IntegrationController.GetIntegrationTestStatusDetailFromSnapshot(snapshot, newIntegrationTestScenario.Name)
+					g.Expect(err).ToNot(HaveOccurred())
+					g.Expect(statusDetail).NotTo(BeNil())
+
+					integrationPipelineRun, err := f.AsKubeDeveloper.IntegrationController.GetIntegrationPipelineRun(newIntegrationTestScenario.Name, snapshot.Name, testNamespace)
+					g.Expect(err).ToNot(HaveOccurred())
+					g.Expect(integrationPipelineRun).NotTo(BeNil())
+
+					g.Expect(statusDetail.TestPipelineRunName).To(Equal(integrationPipelineRun.Name))
+				}, timeout, interval).Should(Succeed())
+			})
+
+			It("checks if snapshot is still marked as failed", func() {
+				snapshot, err = f.AsKubeAdmin.IntegrationController.GetSnapshot(snapshot.Name, "", "", testNamespace)
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(f.AsKubeAdmin.CommonController.HaveTestsSucceeded(snapshot)).To(BeFalse(), "expected tests to fail for snapshot %s/%s", snapshot.GetNamespace(), snapshot.GetName())
+			})
+
+		})
+
 		It("creates an snapshot of push event", func() {
 			sampleImage := "quay.io/redhat-appstudio/sample-image@sha256:841328df1b9f8c4087adbdcfec6cc99ac8308805dea83f6d415d6fb8d40227c1"
 			snapshotPush, err = f.AsKubeAdmin.IntegrationController.CreateSnapshotWithImage(componentName, applicationName, testNamespace, sampleImage)

tests/integration-service/namespace-backed-environments.go

@@ -2,6 +2,7 @@ package integration
 
 import (
 	"fmt"
+	"github.com/redhat-appstudio/operator-toolkit/metadata"
 	"reflect"
 	"strings"
 	"time"
@@ -33,6 +34,7 @@ var _ = framework.IntegrationServiceSuiteDescribe("Namespace-backed Environment
 	var originalComponent *appstudioApi.Component
 	var snapshot, snapshot_push *appstudioApi.Snapshot
 	var integrationTestScenario *integrationv1beta1.IntegrationTestScenario
+	var newIntegrationTestScenario *integrationv1beta1.IntegrationTestScenario
 	var env, ephemeralEnvironment, userPickedEnvironment *appstudioApi.Environment
 	var dtcl *appstudioApi.DeploymentTargetClaimList
 	var dtl *appstudioApi.DeploymentTargetList
@@ -260,6 +262,109 @@ var _ = framework.IntegrationServiceSuiteDescribe("Namespace-backed Environment
 				Expect(err.Error()).To(ContainSubstring(constants.EphemeralEnvAbsenceErrorString))
 			})
 		})
+
+		It("creates a new IntegrationTestScenario with ephemeral environment", func() {
+			var err error
+			newIntegrationTestScenario, err = f.AsKubeAdmin.IntegrationController.CreateIntegrationTestScenarioWithEnvironment(applicationName, testNamespace, gitURL, revisionForNBE, pathInRepoForNBE, userPickedEnvironment)
+			Expect(err).ShouldNot(HaveOccurred())
+		})
+
+		It("updates the Snapshot with the re-run label for the new scenario", FlakeAttempts(3), func() {
+			updatedSnapshot := snapshot.DeepCopy()
+			err := metadata.AddLabels(updatedSnapshot, map[string]string{snapshotRerunLabel: newIntegrationTestScenario.Name})
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(f.AsKubeAdmin.IntegrationController.PatchSnapshot(snapshot, updatedSnapshot)).Should(Succeed())
+			Expect(metadata.GetLabelsWithPrefix(updatedSnapshot, snapshotRerunLabel)).NotTo(BeEmpty())
+		})
+
+		When("An snapshot is updated with a re-run label for a given scenario", func() {
+			It("checks if the re-run label was removed from the Snapshot", func() {
+				Eventually(func() error {
+					snapshot, err = f.AsKubeAdmin.IntegrationController.GetSnapshot(snapshot.Name, "", "", testNamespace)
+					if err != nil {
+						return fmt.Errorf("encountered error while getting Snapshot %s/%s: %w", snapshot.Name, snapshot.Namespace, err)
+					}
+
+					if metadata.HasLabel(snapshot, snapshotRerunLabel) {
+						return fmt.Errorf("the Snapshot %s/%s shouldn't contain the %s label", snapshot.Name, snapshot.Namespace, snapshotRerunLabel)
+					}
+					return nil
+				}, time.Minute*2, time.Second*5).Should(Succeed())
+			})
+
+			It("creates an Ephemeral Environment", func() {
+				Eventually(func() error {
+					ephemeralEnvironment, err = f.AsKubeAdmin.GitOpsController.GetEphemeralEnvironment(snapshot.Spec.Application, snapshot.Name, newIntegrationTestScenario.Name, testNamespace)
+					return err
+				}, time.Minute*3, time.Second*1).Should(Succeed(), fmt.Sprintf("timed out when waiting for the creation of Ephemeral Environment related to snapshot %s", snapshot.Name))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(ephemeralEnvironment.Name).ToNot(BeEmpty())
+			})
+
+			It("checks for SEB after Ephemeral env has been created", func() {
+				seb, err = f.AsKubeAdmin.CommonController.GetSnapshotEnvironmentBinding(applicationName, testNamespace, ephemeralEnvironment)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(seb).ToNot(BeNil())
+				Expect(seb.Spec.Snapshot).To(Equal(snapshot.Name))
+				Expect(seb.Spec.Application).To(Equal(applicationName))
+				Expect(seb.Spec.Environment).To(Equal(ephemeralEnvironment.Name))
+				Expect(seb.Spec.Components).ToNot(BeEmpty())
+			})
+
+			It("checks if the new integration pipelineRun started", Label("slow"), func() {
+				reRunPipelineRun, err := f.AsKubeDeveloper.IntegrationController.WaitForIntegrationPipelineToGetStarted(newIntegrationTestScenario.Name, snapshot.Name, testNamespace)
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(reRunPipelineRun).ShouldNot(BeNil())
+
+				Expect(reRunPipelineRun.Labels[snapshotAnnotation]).To(ContainSubstring(snapshot.Name))
+				Expect(reRunPipelineRun.Labels[scenarioAnnotation]).To(ContainSubstring(newIntegrationTestScenario.Name))
+				Expect(reRunPipelineRun.Labels[environmentLabel]).To(ContainSubstring(ephemeralEnvironment.Name))
+			})
+
+			It("checks if all integration pipelineRuns finished successfully", Label("slow"), func() {
+				Expect(f.AsKubeDeveloper.IntegrationController.WaitForAllIntegrationPipelinesToBeFinished(testNamespace, applicationName, snapshot)).To(Succeed())
+			})
+
+			It("checks if the name of the re-triggered pipelinerun is reported in the Snapshot", FlakeAttempts(3), func() {
+				Eventually(func(g Gomega) {
+					snapshot, err = f.AsKubeAdmin.IntegrationController.GetSnapshot(snapshot.Name, "", "", testNamespace)
+					g.Expect(err).ShouldNot(HaveOccurred())
+
+					statusDetail, err := f.AsKubeDeveloper.IntegrationController.GetIntegrationTestStatusDetailFromSnapshot(snapshot, newIntegrationTestScenario.Name)
+					g.Expect(err).ToNot(HaveOccurred())
+					g.Expect(statusDetail).NotTo(BeNil())
+
+					integrationPipelineRun, err := f.AsKubeDeveloper.IntegrationController.GetIntegrationPipelineRun(newIntegrationTestScenario.Name, snapshot.Name, testNamespace)
+					g.Expect(err).ToNot(HaveOccurred())
+					g.Expect(integrationPipelineRun).NotTo(BeNil())
+
+					g.Expect(statusDetail.TestPipelineRunName).To(Equal(integrationPipelineRun.Name))
+				}, time.Minute*2, time.Second*5).Should(Succeed())
+			})
+
+			It("checks if snapshot is still marked as successful", func() {
+				snapshot, err = f.AsKubeAdmin.IntegrationController.GetSnapshot(snapshot.Name, "", "", testNamespace)
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(f.AsKubeAdmin.CommonController.HaveTestsSucceeded(snapshot)).To(BeTrue(), "expected tests to succeed for snapshot %s/%s", snapshot.GetNamespace(), snapshot.GetName())
+			})
+
+			It("should lead to SnapshotEnvironmentBinding getting deleted", func() {
+				Eventually(func() error {
+					_, err = f.AsKubeAdmin.CommonController.GetSnapshotEnvironmentBinding(applicationName, testNamespace, ephemeralEnvironment)
+					return err
+				}, time.Minute*3, time.Second*5).ShouldNot(Succeed(), fmt.Sprintf("timed out when waiting for SnapshotEnvironmentBinding to be deleted for application %s/%s", testNamespace, applicationName))
+				Expect(err.Error()).To(ContainSubstring(constants.SEBAbsenceErrorString))
+			})
+
+			It("should lead to ephemeral environment getting deleted", func() {
+				Eventually(func() error {
+					ephemeralEnvironment, err = f.AsKubeAdmin.GitOpsController.GetEphemeralEnvironment(snapshot.Spec.Application, snapshot.Name, newIntegrationTestScenario.Name, testNamespace)
+					return err
+				}, time.Minute*3, time.Second*1).ShouldNot(Succeed(), fmt.Sprintf("timed out when waiting for the Ephemeral Environment %s to be deleted", ephemeralEnvironment.Name))
+				Expect(err.Error()).To(ContainSubstring(constants.EphemeralEnvAbsenceErrorString))
+			})
+
+		})
 	})
 
 	Describe("when valid DeploymentTargetClass doesn't exist", Ordered, func() {