From 83a04519893cf7e372ad716c044944f242385e29 Mon Sep 17 00:00:00 2001 From: Jing Qi Date: Wed, 20 Dec 2023 18:08:59 +0800 Subject: [PATCH] fix: issues in release-pipelines test suite Signed-off-by: Jing Qi * Fixed the additionalComponentName in mapping and changed to use development branch as default branch and updated the name of pipeline * Added the envirionment variables required by release-pipelines-suite * Updated some FBC constants to variables to make the test repeatable with failure. --- magefiles/magefile.go | 8 +- pkg/constants/constants.go | 9 +- tests/release/const.go | 4 +- tests/release/pipelines/fbc_release.go | 93 +++++++++++-------- ...try.go => rh_push_to_external_registry.go} | 22 +++-- 5 files changed, 81 insertions(+), 55 deletions(-) rename tests/release/pipelines/{push_to_external_registry.go => rh_push_to_external_registry.go} (96%) diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 3fdfceec7..949ee7c56 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -494,9 +494,15 @@ func (ci CI) setRequiredEnvVars() error { envVarPrefix := "RELEASE_SERVICE" // "rehearse" jobs metadata are not relevant for testing if !strings.Contains(jobName, "rehearse") { - os.Setenv(fmt.Sprintf("%s_CATALOG_URL", envVarPrefix), fmt.Sprintf("https://github.com/%s/%s", pr.Organization, pr.RepoName)) + os.Setenv(fmt.Sprintf("%s_CATALOG_URL", envVarPrefix), fmt.Sprintf("https://github.com/%s/%s", pr.RemoteName, pr.RepoName)) os.Setenv(fmt.Sprintf("%s_CATALOG_REVISION", envVarPrefix), pr.CommitSHA) } + if os.Getenv("REL_IMAGE_CONTROLLER_QUAY_ORG") != "" { + os.Setenv("IMAGE_CONTROLLER_QUAY_ORG", os.Getenv("REL_IMAGE_CONTROLLER_QUAY_ORG")) + } + if os.Getenv("REL_IMAGE_CONTROLLER_QUAY_TOKEN") != "" { + os.Setenv("IMAGE_CONTROLLER_QUAY_TOKEN", os.Getenv("REL_IMAGE_CONTROLLER_QUAY_TOKEN")) + } os.Setenv("E2E_TEST_SUITE_LABEL", "release-pipelines") } else { // openshift/release rehearse job for e2e-tests/infra-deployments repos requiresMultiPlatformTests = true diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 238e2e044..e45b2ad6f 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -128,9 +128,9 @@ const ( ReleasePipelineImageRef = "quay.io/hacbs-release/pipeline-release:0.20" - FromIndex = "quay.io/scoheb/fbc-index-testing:latest" - TargetIndex = "quay.io/scoheb/fbc-target-index-testing:latest" - BinaryImage = "registry.redhat.io/openshift4/ose-operator-registry:v4.12" + FromIndex = "registry-proxy.engineering.redhat.com/rh-osbs/iib-preview-rhtap:v4.13" + TargetIndex = "quay.io/redhat/redhat----preview-operator-index:v4.13" + BinaryImage = "registry.redhat.io/openshift4/ose-operator-registry:v4.13" StrategyConfigsRepo = "strategy-configs" StrategyConfigsDefaultBranch = "main" @@ -168,6 +168,9 @@ const ( // #app-studio-ci-reports channel id SlackCIReportsChannelID = "C02M210JZ7B" + + DevReleaseTeam = "dev-release-team" + ManagedReleaseTeam = "managed-release-team" ) var ( diff --git a/tests/release/const.go b/tests/release/const.go index d3d043e31..ab30be73c 100644 --- a/tests/release/const.go +++ b/tests/release/const.go @@ -24,7 +24,7 @@ const ( ReleaseCreationTimeout = 5 * time.Minute ReleasePipelineRunCreationTimeout = 10 * time.Minute - ReleasePipelineRunCompletionTimeout = 20 * time.Minute + ReleasePipelineRunCompletionTimeout = 60 * time.Minute ReleasePlanStatusUpdateTimeout = 1 * time.Minute DefaultInterval = 100 * time.Millisecond @@ -48,5 +48,5 @@ var ManagednamespaceSecret = []corev1.ObjectReference{ // Pipelines variables var ( RelSvcCatalogURL string = utils.GetEnv("RELEASE_SERVICE_CATALOG_URL", "https://github.com/redhat-appstudio/release-service-catalog") - RelSvcCatalogRevision string = utils.GetEnv("RELEASE_SERVICE_CATALOG_REVISION", "main") + RelSvcCatalogRevision string = utils.GetEnv("RELEASE_SERVICE_CATALOG_REVISION", "development") ) diff --git a/tests/release/pipelines/fbc_release.go b/tests/release/pipelines/fbc_release.go index ad8f36608..9d04742f0 100644 --- a/tests/release/pipelines/fbc_release.go +++ b/tests/release/pipelines/fbc_release.go @@ -6,6 +6,7 @@ import ( "os" "time" + "github.com/devfile/library/v2/pkg/util" ecp "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -21,31 +22,22 @@ import ( ) const ( - fbcApplicationName = "fbc-pipelines-aplication" - fbcHotfixAppName = "fbc-hotfix-aplication" - fbcComponentName = "fbc-pipelines-component" - fbcHotfixCompName = "fbc-hotfix-component" - fbcReleasePlanName = "fbc-pipelines-releaseplan" - fbcHotfixRPName = "fbc-hotfix-releaseplan" - fbcReleasePlanAdmissionName = "fbc-pipelines-releaseplanadmission" - fbcHotfixRPAName = "fbc-hotfix-releaseplanadmission" - fbcEnterpriseContractPolicyName = "fbc-pipelines-policy" - fbcHotfixECPolicyName = "fbc-hotfix-policy" - fbcServiceAccountName = "release-service-account" - fbcSourceGitURL = "https://github.com/redhat-appstudio-qe/fbc-sample-repo" - targetPort = 50051 - relSvcCatalogPathInRepo = "pipelines/fbc-release/fbc-release.yaml" - ecPolicyLibPath = "github.com/enterprise-contract/ec-policies//policy/lib" - ecPolicyReleasePath = "github.com/enterprise-contract/ec-policies//policy/release" - ecPolicyDataBundle = "oci::quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles:latest" - ecPolicyDataPath = "github.com/release-engineering/rhtap-ec-policy//data" + fbcServiceAccountName = "release-service-account" + fbcSourceGitURL = "https://github.com/redhat-appstudio-qe/fbc-sample-repo" + targetPort = 50051 + relSvcCatalogPathInRepo = "pipelines/fbc-release/fbc-release.yaml" + ecPolicyLibPath = "github.com/enterprise-contract/ec-policies//policy/lib" + ecPolicyReleasePath = "github.com/enterprise-contract/ec-policies//policy/release" + ecPolicyDataBundle = "oci::quay.io/redhat-appstudio-tekton-catalog/data-acceptable-bundles:latest" + ecPolicyDataPath = "github.com/release-engineering/rhtap-ec-policy//data" ) var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release-pipelines", "fbc-tests"), func() { defer GinkgoRecover() - var devWorkspace = os.Getenv(constants.RELEASE_DEV_WORKSPACE_ENV) - var managedWorkspace = os.Getenv(constants.RELEASE_MANAGED_WORKSPACE_ENV) + var devWorkspace = utils.GetEnv(constants.RELEASE_DEV_WORKSPACE_ENV, constants.DevReleaseTeam) + var managedWorkspace = utils.GetEnv(constants.RELEASE_MANAGED_WORKSPACE_ENV, constants.ManagedReleaseTeam) + var devNamespace = devWorkspace + "-tenant" var managedNamespace = managedWorkspace + "-tenant" @@ -53,6 +45,16 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release- var devFw *framework.Framework var managedFw *framework.Framework var issueId = "bz12345" + var fbcApplicationName = "fbc-pipelines-app-" + util.GenerateRandomString(4) + var fbcHotfixAppName = "fbc-hotfix-app-" + util.GenerateRandomString(4) + var fbcComponentName = "fbc-pipelines-comp-" + util.GenerateRandomString(4) + var fbcHotfixCompName = "fbc-hotfix-comp-" + util.GenerateRandomString(4) + var fbcReleasePlanName = "fbc-pipelines-rp-" + util.GenerateRandomString(4) + var fbcHotfixRPName = "fbc-hotfix-rp-" + util.GenerateRandomString(4) + var fbcReleasePlanAdmissionName = "fbc-pipelines-rpa-" + util.GenerateRandomString(4) + var fbcHotfixRPAName = "fbc-hotfix-rpa-" + util.GenerateRandomString(4) + var fbcEnterpriseContractPolicyName = "fbc-pipelines-policy-" + util.GenerateRandomString(4) + var fbcHotfixECPolicyName = "fbc-hotfix-policy-" + util.GenerateRandomString(4) AfterEach(framework.ReportFailure(&devFw)) @@ -81,6 +83,10 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release- Expect(err).NotTo(HaveOccurred()) managedNamespace = managedFw.UserNamespace + // Linking the build secret to the pipeline service account in dev namespace. + err = devFw.AsKubeAdmin.CommonController.LinkSecretToServiceAccount(devNamespace, releasecommon.HacbsReleaseTestsTokenSecret, constants.DefaultPipelineServiceAccount, true) + Expect(err).ToNot(HaveOccurred()) + _, err = devFw.AsKubeDeveloper.HasController.CreateApplication(fbcApplicationName, devNamespace) Expect(err).NotTo(HaveOccurred()) @@ -90,6 +96,7 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release- createFBCReleasePlanAdmission(fbcReleasePlanAdmissionName, *managedFw, devNamespace, managedNamespace, fbcApplicationName, fbcEnterpriseContractPolicyName, relSvcCatalogPathInRepo, "false", "") component = releasecommon.CreateComponentByCDQ(*devFw, devNamespace, managedNamespace, fbcApplicationName, fbcComponentName, fbcSourceGitURL) createFBCEnterpriseContractPolicy(fbcEnterpriseContractPolicyName, *managedFw, devNamespace, managedNamespace) + }) AfterAll(func() { @@ -139,7 +146,7 @@ var _ = framework.ReleasePipelinesSuiteDescribe("FBC e2e-tests", Label("release- _, err = devFw.AsKubeDeveloper.ReleaseController.CreateReleasePlan(fbcHotfixRPName, devNamespace, fbcHotfixAppName, managedNamespace, "true") Expect(err).NotTo(HaveOccurred()) - createFBCReleasePlanAdmission(fbcHotfixRPAName, *managedFw, devNamespace, managedNamespace,fbcHotfixAppName, fbcHotfixECPolicyName, relSvcCatalogPathInRepo, "true", issueId) + createFBCReleasePlanAdmission(fbcHotfixRPAName, *managedFw, devNamespace, managedNamespace, fbcHotfixAppName, fbcHotfixECPolicyName, relSvcCatalogPathInRepo, "true", issueId) component = releasecommon.CreateComponentByCDQ(*devFw, devNamespace, managedNamespace, fbcHotfixAppName, fbcHotfixCompName, fbcSourceGitURL) createFBCEnterpriseContractPolicy(fbcHotfixECPolicyName, *managedFw, devNamespace, managedNamespace) }) @@ -181,16 +188,20 @@ func assertReleasePipelineRunSucceeded(devFw, managedFw framework.Framework, dev Eventually(func() error { snapshot, err := devFw.AsKubeDeveloper.IntegrationController.GetSnapshot("", buildPr.Name, "", devNamespace) if err != nil { - return fmt.Errorf("snapshot %s in namespace %s has not been found yet", snapshot.Name, devNamespace) + return fmt.Errorf("snapshot in namespace %s has not been found yet", devNamespace) } releaseCR, err := devFw.AsKubeDeveloper.ReleaseController.GetRelease("", snapshot.Name, devNamespace) if err != nil { - return fmt.Errorf("release %s in namespace %s has not been found yet", releaseCR.Name, managedNamespace) + return fmt.Errorf("release in namespace %s has not been found yet", managedNamespace) } Expect(err).ShouldNot(HaveOccurred()) releasePr, err := managedFw.AsKubeAdmin.ReleaseController.GetPipelineRunInNamespace(managedFw.UserNamespace, releaseCR.GetName(), releaseCR.GetNamespace()) + if err != nil { + return fmt.Errorf("releasePipelineRun in namespace %s has not been found yet", managedNamespace) + } Expect(err).ShouldNot(HaveOccurred()) + if !releasePr.IsDone() { return fmt.Errorf("release pipelinerun %s in namespace %s did not finish yet", releasePr.Name, releasePr.Namespace) } @@ -203,9 +214,13 @@ func assertReleasePipelineRunSucceeded(devFw, managedFw framework.Framework, dev func assertReleaseCRSucceeded(devFw framework.Framework, devNamespace, managedNamespace, fbcAppName string, component *appservice.Component) { Eventually(func() error { buildPr, err := devFw.AsKubeDeveloper.HasController.GetComponentPipelineRun(component.Name, fbcAppName, devNamespace, "") - Expect(err).ShouldNot(HaveOccurred()) - snapshot, err := devFw.AsKubeDeveloper.IntegrationController.GetSnapshot("", buildPr.Name , "", devNamespace) - Expect(err).ShouldNot(HaveOccurred()) + if err != nil { + return err + } + snapshot, err := devFw.AsKubeDeveloper.IntegrationController.GetSnapshot("", buildPr.Name, "", devNamespace) + if err != nil { + return err + } releaseCR, err := devFw.AsKubeDeveloper.ReleaseController.GetRelease("", snapshot.Name, devNamespace) if err != nil { return err @@ -218,7 +233,7 @@ func assertReleaseCRSucceeded(devFw framework.Framework, devNamespace, managedNa }, releasecommon.ReleaseCreationTimeout, releasecommon.DefaultInterval).Should(Succeed()) } -func createFBCEnterpriseContractPolicy(fbcECPName string,managedFw framework.Framework, devNamespace, managedNamespace string) { +func createFBCEnterpriseContractPolicy(fbcECPName string, managedFw framework.Framework, devNamespace, managedNamespace string) { defaultEcPolicySpec := ecp.EnterpriseContractPolicySpec{ Description: "Red Hat's enterprise requirements", PublicKey: "k8s://openshift-pipelines/public-key", @@ -228,8 +243,8 @@ func createFBCEnterpriseContractPolicy(fbcECPName string,managedFw framework.Fra Data: []string{ecPolicyDataBundle, ecPolicyDataPath}, }}, Configuration: &ecp.EnterpriseContractPolicyConfiguration{ - Exclude: []string{"cve", "step_image_registries", "tasks.required_tasks_found:prefetch-dependencies"}, - Include: []string{"minimal"}, + Exclude: []string{"cve", "step_image_registries", "tasks.required_tasks_found:prefetch-dependencies"}, + Include: []string{"minimal", "slsa3"}, }, } @@ -242,16 +257,16 @@ func createFBCReleasePlanAdmission(fbcRPAName string, managedFw framework.Framew var err error data, err := json.Marshal(map[string]interface{}{ "fbc": map[string]interface{}{ - "fromIndex": constants.FromIndex, - "targetIndex": constants.TargetIndex, - "binaryImage": constants.BinaryImage, - "publishingCredentials": "fbc-preview-publishing-credentials", - "iibServiceConfigSecret": "iib-preview-services-config", - "iibOverwriteFromIndexCredential": "iib-preview-overwritefromimage-credential", - "requestUpdateTimeout": "420", - "buildTimeoutSeconds": "480", - "hotfix": hotfix, - "issueId": issueId, + "fromIndex": constants.FromIndex, + "targetIndex": constants.TargetIndex, + "binaryImage": constants.BinaryImage, + "publishingCredentials": "fbc-preview-publishing-credentials", + "iibServiceConfigSecret": "iib-preview-services-config", + "iibOverwriteFromIndexCredential": "iib-overwrite-fromimage-credentials", + "requestUpdateTimeout": "420", + "buildTimeoutSeconds": "480", + "hotfix": hotfix, + "issueId": issueId, }, "sign": map[string]interface{}{ "configMapName": "hacbs-signing-pipeline-config-redhatbeta2", diff --git a/tests/release/pipelines/push_to_external_registry.go b/tests/release/pipelines/rh_push_to_external_registry.go similarity index 96% rename from tests/release/pipelines/push_to_external_registry.go rename to tests/release/pipelines/rh_push_to_external_registry.go index 5b73fc662..8a6f822cf 100644 --- a/tests/release/pipelines/push_to_external_registry.go +++ b/tests/release/pipelines/rh_push_to_external_registry.go @@ -30,7 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var _ = framework.ReleasePipelinesSuiteDescribe("[HACBS-1571]test-release-e2e-push-image-to-pyxis", Label("release-pipelines", "pushPyxis", "HACBS"), Pending, func() { +var _ = framework.ReleasePipelinesSuiteDescribe("[HACBS-1571]test-release-e2e-push-image-to-pyxis", Label("release-pipelines", "pushPyxis", "HACBS"), func() { defer GinkgoRecover() // Initialize the tests controllers var fw *framework.Framework @@ -82,6 +82,7 @@ var _ = framework.ReleasePipelinesSuiteDescribe("[HACBS-1571]test-release-e2e-pu // Creating k8s secret to access Pyxis stage based on base64 decoded of key and cert pyxisKeyDecoded, err = base64.StdEncoding.DecodeString(string(keyPyxisStage)) Expect(err).ToNot(HaveOccurred()) + pyxisCertDecoded, err = base64.StdEncoding.DecodeString(string(certPyxisStage)) Expect(err).ToNot(HaveOccurred()) @@ -139,20 +140,18 @@ var _ = framework.ReleasePipelinesSuiteDescribe("[HACBS-1571]test-release-e2e-pu additionalComponentDetected = compDetected } - _, err = fw.AsKubeAdmin.ReleaseController.CreateReleasePlan(releaseConst.SourceReleasePlanName, devNamespace, releaseConst.ApplicationNameDefault, managedNamespace, "") + _, err = fw.AsKubeAdmin.ReleaseController.CreateReleasePlan(releaseConst.SourceReleasePlanName, devNamespace, releaseConst.ApplicationNameDefault, managedNamespace, "true") Expect(err).NotTo(HaveOccurred()) data, err := json.Marshal(map[string]interface{}{ "mapping": map[string]interface{}{ "components": []map[string]interface{}{ { - "name": compName, - //"repository": "quay.io/redhat-appstudio-qe/dcmetromap", + "name": compName, "repository": "quay.io/" + utils.GetQuayIOOrganization() + "/dcmetromap", }, { - "name": releaseConst.AdditionalComponentName, - //"repository": "quay.io/redhat-appstudio-qe/simplepython", + "name": additionalCompName, "repository": "quay.io/" + utils.GetQuayIOOrganization() + "/simplepython", }, }, @@ -164,12 +163,13 @@ var _ = framework.ReleasePipelinesSuiteDescribe("[HACBS-1571]test-release-e2e-pu }) Expect(err).NotTo(HaveOccurred()) - _, err = fw.AsKubeAdmin.ReleaseController.CreateReleasePlanAdmission(releaseConst.TargetReleasePlanAdmissionName, managedNamespace, releaseConst.ReleaseEnvironment, devNamespace, releaseConst.ReleaseStrategyPolicyDefault, releaseConst.ReleasePipelineServiceAccountDefault, []string{releaseConst.ApplicationNameDefault}, true, &tektonutils.PipelineRef{ + _, err = fw.AsKubeAdmin.ReleaseController.CreateReleasePlanAdmission(releaseConst.TargetReleasePlanAdmissionName, managedNamespace, "", devNamespace, releaseConst.ReleaseStrategyPolicyDefault, releaseConst.ReleasePipelineServiceAccountDefault, []string{releaseConst.ApplicationNameDefault}, true, &tektonutils.PipelineRef{ Resolver: "git", Params: []tektonutils.Param{ {Name: "url", Value: releaseConst.RelSvcCatalogURL}, - {Name: "revision", Value: releaseConst.RelSvcCatalogRevision}, - {Name: "pathInRepo", Value: "pipelines/push-to-external-registry/push-to-external-registry.yaml"}, + // {Name: "revision", Value: releaseConst.RelSvcCatalogRevision}, + {Name: "revision", Value: "development"}, + {Name: "pathInRepo", Value: "pipelines/rh-push-to-external-registry/rh-push-to-external-registry.yaml"}, }, }, &runtime.RawExtension{ Raw: data, @@ -286,7 +286,9 @@ var _ = framework.ReleasePipelinesSuiteDescribe("[HACBS-1571]test-release-e2e-pu var errMsg string for _, pr := range []*v1beta1.PipelineRun{releasePR1, releasePR2} { pr, err = fw.AsKubeAdmin.TektonController.GetPipelineRun(pr.GetName(), pr.GetNamespace()) - Expect(err).ShouldNot(HaveOccurred()) + if err != nil { + return err + } Expect(tekton.HasPipelineRunFailed(pr)).ToNot(BeTrue(), fmt.Sprintf("Release PipelineRun %s/%s failed", pr.GetNamespace(), pr.GetName())) if pr.IsDone() { Expect(tekton.HasPipelineRunSucceeded(pr)).To(BeTrue(), fmt.Sprintf("Release PipelineRun %s/%s did not succceed", pr.GetNamespace(), pr.GetName()))