(v1.0.4-release) backport AQA_Test_Pipeline enhancements (#5533)

* Improve AQA_Test_Pipeline for release testing (#5512) related: #5510 fixes: #5509 fixes: eclipse-openj9/openj9#20004 Signed-off-by: Lan Xia <[email protected]> * Limit FIPS functional testing on OpenJcePlusTests and security related: backlog/issues/1494 Signed-off-by: Lan Xia <[email protected]> --------- Signed-off-by: Lan Xia <[email protected]>
adoptium · Aug 29, 2024 · 6280d2a · 6280d2a
1 parent 48490ac
commit 6280d2a
Show file tree

Hide file tree

Showing 3 changed files with 204 additions and 32 deletions.
diff --git a/buildenv/jenkins/aqaTestPipeline.groovy b/buildenv/jenkins/aqaTestPipeline.groovy
@@ -6,10 +6,17 @@ def TARGETS = params.TARGETS ?: "Grinder"
 TARGETS = TARGETS.trim().split("\\s*,\\s*")
 
 def PARALLEL = params.PARALLEL ? params.PARALLEL : "Dynamic"
-def NUM_MACHINES = params.NUM_MACHINES ? params.NUM_MACHINES : 3
+
+def NUM_MACHINES = ""
+if (params.NUM_MACHINES) {
+    NUM_MACHINES = params.NUM_MACHINES
+} else if (!params.TEST_TIME && PARALLEL == "Dynamic") {
+    // set default NUM_MACHINES to 3 if params.NUM_MACHINES and params.TEST_TIME are not set and PARALLEL is Dynamic 
+    NUM_MACHINES = 3
+}
 def SDK_RESOURCE = params.SDK_RESOURCE ? params.SDK_RESOURCE : "releases"
 def TIME_LIMIT = params.TIME_LIMIT ? params.TIME_LIMIT : 10
-def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN : false
+def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN.toBoolean() : false
 def TRSS_URL = params.TRSS_URL ? params.TRSS_URL : "https://trss.adoptium.net/"
 def TEST_FLAG = (params.TEST_FLAG) ?: ""
 def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
@@ -18,20 +25,62 @@ def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
 def DEFAULT_SUFFIX = (env.BUILD_USER_ID) ? "${env.BUILD_USER_ID} - jdk-${params.JDK_VERSIONS}" : "jdk-${params.JDK_VERSIONS}"
 def PIPELINE_DISPLAY_NAME = (params.PIPELINE_DISPLAY_NAME) ? "#${currentBuild.number} - ${params.PIPELINE_DISPLAY_NAME}" : "#${currentBuild.number} - ${DEFAULT_SUFFIX}"
 
-def JOBS = [:]
-
 // Set the AQA_TEST_PIPELINE Jenkins job displayName
 currentBuild.setDisplayName(PIPELINE_DISPLAY_NAME)
 
-def suffix = ""
-if (TEST_FLAG) {
-    suffix = "_" + TEST_FLAG.toLowerCase().trim()
-}
+def defaultTestTargets = "sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf"
+def defaultFipsTestTargets = "extended.functional,sanity.openjdk,extended.openjdk,sanity.jck,extended.jck,special.jck"
+
+JOBS = [:]
+fail = false
 
-def fail = false
-int jobNum = 0
 JDK_VERSIONS.each { JDK_VERSION ->
-    PLATFORMS.each { PLATFORM ->
+    if (params.PLATFORMS == "release") {
+        def configJson = []
+        node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
+            checkout scm
+            dir (env.WORKSPACE) {
+                def filePath = "./aqa-tests/buildenv/jenkins/config/${params.VARIANT}/"
+                filePath = filePath + "default.json"
+                if (fileExists(filePath + "jdk${JDK_VERSION}.json")) {
+                    filePath = filePath + "jdk${JDK_VERSION}.json"
+                }
+                configJson = readJSON(file: filePath)
+            }
+        }
+
+        configJson.each { item ->
+            def releaseTestFlag = ""
+            releaseTestFlag = item.TEST_FLAG
+            item.PLATFORM_TARGETS.each { pt ->
+                pt.each{ p, t ->
+                    def releasePlatform = p
+                    def releaseTargets = ""
+                    if (t.contains("defaultFipsTestTargets")) {
+                        releaseTargets = t.replace("defaultFipsTestTargets","${defaultFipsTestTargets}")
+                    } else {
+                        releaseTargets = t.replace("defaultTestTargets","${defaultTestTargets}")
+                    }
+                    String[] releasePlatformArray = releasePlatform.split("\\s*,\\s*")
+                    String[] releaseTargetsArray = releaseTargets.split("\\s*,\\s*")
+                    generateJobs(JDK_VERSION, releaseTestFlag, releasePlatformArray, releaseTargetsArray)
+                }
+            }
+        }
+    } else {
+        generateJobs(JDK_VERSION, TEST_FLAG, PLATFORMS, TARGETS)
+    }
+}
+parallel JOBS
+if (fail) {
+    currentBuild.result = "FAILURE"
+}
+
+def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) {
+    if (jobTestFlag == "NONE") {
+        jobTestFlag = ""
+    }
+    jobPlatforms.each { PLATFORM ->
         String[] tokens = PLATFORM.split('_')
         def os = tokens[1];
         def arch = tokens[0];
@@ -63,30 +112,83 @@ JDK_VERSIONS.each { JDK_VERSION ->
             if (params.VARIANT == "openj9") {
                 // get IBM Semeru CE
                 sdk_resource_value = "customized"
-                download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
+                download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
             }
         }
         echo "download_url: ${download_url}"
 
-        TARGETS.each { TARGET ->
+        jobTargets.each { TARGET ->
             def TEST_JOB_NAME = "Grinder"
             if (TARGET.contains("Grinder")) {
                 TEST_JOB_NAME = TARGET
             } else {
-                TEST_JOB_NAME = "Test_openjdk${JDK_VERSION}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
+                def suffix = ""
+                if (jobTestFlag) {
+                    suffix = "_" + jobTestFlag.toLowerCase().trim()
+                }
+                TEST_JOB_NAME = "Test_openjdk${jobJdkVersion}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
             }
             echo "TEST_JOB_NAME: ${TEST_JOB_NAME}"
 
             def keep_reportdir = false
-            if (TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
+            if (TARGET.contains("functional") || TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
                 keep_reportdir = true
             }
-            if (TARGET.contains("functional") || TARGET.contains("perf")) {
-                if (params.VARIANT == "temurin") {
-                    PARALLEL = "None"
+
+            def DYNAMIC_COMPILE = false
+            if (!params.DYNAMIC_COMPILE) {
+                if (("${TARGET}".contains('functional')) || ("${TARGET}".contains('external'))) {
+                    DYNAMIC_COMPILE = true
+                } else {
+                    DYNAMIC_COMPILE = false
                 }
+            } else {
+                DYNAMIC_COMPILE = params.DYNAMIC_COMPILE ? params.DYNAMIC_COMPILE.toBoolean() : false
             }
 
+            def VENDOR_TEST_REPOS = ''
+            def VENDOR_TEST_BRANCHES = ''
+            def VENDOR_TEST_DIRS = ''
+            int rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 0
+            def buildList = params.BUILD_LIST ?: ""
+            if (params.VARIANT == "openj9") {
+                // default rerunIterations is 3 for openj9
+                rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 3
+                if (TARGET.contains('external')) {
+                    PARALLEL = "None"
+                    rerunIterations = 0
+                } else if (TARGET.contains('functional')) {
+                    if (jobTestFlag.contains("FIPS")) {
+                        if (!buildList) {
+                            buildList = "functional/OpenJcePlusTests,functional/security"
+                        }
+                    } else {
+                        VENDOR_TEST_REPOS = '[email protected]:runtimes/test.git'
+                        VENDOR_TEST_BRANCHES = params.ADOPTOPENJDK_BRANCH ?: 'master'
+                        VENDOR_TEST_DIRS = 'functional'
+                    }
+                } else if (TARGET.contains('jck')) {
+                    VENDOR_TEST_REPOS = '[email protected]:runtimes/jck.git'
+                    VENDOR_TEST_BRANCHES = "main"
+                    VENDOR_TEST_DIRS = 'jck'
+                } else if (TARGET.contains('openjdk')) {
+                    // only use osb repo for regular testing
+                    if (TARGET.contains('special') && jobTestFlag == "") {
+                        VENDOR_TEST_REPOS = '[email protected]:runtimes/osb-tests.git'
+                        VENDOR_TEST_BRANCHES = "ibm_tlda"
+                        VENDOR_TEST_DIRS = 'openjdk'
+                    }
+                }
+
+                if (jobTestFlag.contains("FIPS") || (TARGET.contains("dev"))) {
+                    rerunIterations = 0
+                }
+            } else if (params.VARIANT == "temurin") {
+                if (TARGET.contains("functional") || TARGET.contains("perf")) {
+                    PARALLEL = "None"
+                }
+            }
+            echo "AUTO_AQA_GEN: ${AUTO_AQA_GEN}"
             // Grinder job has special settings and should be regenerated specifically, not via aqaTestPipeline
             if (AUTO_AQA_GEN && !TEST_JOB_NAME.contains("Grinder")) {
                 String[] targetTokens = TARGET.split("\\.")
@@ -96,7 +198,7 @@ JDK_VERSIONS.each { JDK_VERSION ->
                     string(name: 'TEST_JOB_NAME', value: TEST_JOB_NAME),
                     string(name: 'LEVELS', value: level),
                     string(name: 'GROUPS', value: group),
-                    string(name: 'JDK_VERSIONS', value: JDK_VERSION),
+                    string(name: 'JDK_VERSIONS', value: jobJdkVersion),
                     string(name: 'ARCH_OS_LIST', value: PLATFORM),
                     string(name: 'JDK_IMPL', value: jdk_impl),
                     booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: LIGHT_WEIGHT_CHECKOUT)
@@ -124,8 +226,8 @@ JDK_VERSIONS.each { JDK_VERSION ->
                         childParams << booleanParam(name: param.key, value: LIGHT_WEIGHT_CHECKOUT.toBoolean())
                     } else if (param.key == "TIME_LIMIT") {
                         childParams << string(name: param.key, value: TIME_LIMIT.toString())
-                    } else if (param.key == "TRSS_URL") {
-                        childParams << string(name: param.key, value: TRSS_URL)
+                    } else if (param.key == "TEST_FLAG") {
+                        childParams << string(name: param.key, value: jobTestFlag)
                     } else if (param.key == "KEEP_REPORTDIR") {
                         childParams << booleanParam(name: param.key, value: keep_reportdir.toBoolean())
                     } else {
@@ -137,17 +239,25 @@ JDK_VERSIONS.each { JDK_VERSION ->
                         }
                     }
                 }
+                childParams << booleanParam(name: "DYNAMIC_COMPILE", value: DYNAMIC_COMPILE.toBoolean())
                 childParams << booleanParam(name: "GENERATE_JOBS", value: AUTO_AQA_GEN.toBoolean())
-                childParams << string(name: "JDK_VERSION", value: JDK_VERSION)
-                childParams << string(name: "PLATFORM", value: PLATFORM)
                 childParams << string(name: "JDK_IMPL", value: jdk_impl)
+                childParams << string(name: "JDK_VERSION", value: jobJdkVersion)
+                childParams << string(name: "PLATFORM", value: PLATFORM)
+                childParams << string(name: "RERUN_ITERATIONS", value: rerunIterations.toString())
+                childParams << string(name: "VENDOR_TEST_BRANCHES", value: VENDOR_TEST_BRANCHES)
+                childParams << string(name: "VENDOR_TEST_DIRS", value: VENDOR_TEST_DIRS)
+                childParams << string(name: "VENDOR_TEST_REPOS", value: VENDOR_TEST_REPOS)
+                if (buildList) {
+                    childParams << string(name: "BUILD_LIST", value: buildList)
+                }
 
-                jobNum++
+                int jobNum = JOBS.size() + 1
                 JOBS["${TEST_JOB_NAME}_${jobNum}"] = {
                     def downstreamJob = build job: TEST_JOB_NAME, parameters: childParams, propagate: false, wait: true
                     def downstreamJobResult = downstreamJob.getResult()
-                    echo " ${TEST_JOB_NAME} result is ${downstreamJobResult}"
-                    if (downstreamJob.getResult() == 'SUCCESS' || downstreamJob.getResult() == 'UNSTABLE') {
+                    echo "${TEST_JOB_NAME} result is ${downstreamJobResult}"
+                    if (downstreamJobResult == 'SUCCESS' || downstreamJobResult == 'UNSTABLE') {
                         echo "[NODE SHIFT] MOVING INTO CONTROLLER NODE..."
                         node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
                             cleanWs disableDeferredWipeout: true, deleteDirs: true
@@ -185,9 +295,4 @@ JDK_VERSIONS.each { JDK_VERSION ->
             }
         }
     }
-}
-parallel JOBS
-if (fail) {
-    currentBuild.result = "FAILURE"
-}
-
+}
diff --git a/buildenv/jenkins/config/openj9/default.json b/buildenv/jenkins/config/openj9/default.json
@@ -0,0 +1,43 @@
+[
+    {
+        "TEST_FLAG" : "NONE", 
+        "PLATFORM_TARGETS" : [
+            { "aarch64_linux" : "defaultTestTargets,dev.external" },
+            { "aarch64_mac" : "defaultTestTargets" },
+            { "ppc64_aix" : "defaultTestTargets" },
+            { "ppc64le_linux" : "defaultTestTargets,dev.external" },
+            { "s390x_linux" : "defaultTestTargets,dev.external" },
+            { "x86-64_linux" : "defaultTestTargets,dev.external" },
+            { "x86-64_mac" : "defaultTestTargets" },
+            { "x86-64_windows" : "defaultTestTargets" }
+        ]
+    },
+    {
+        "TEST_FLAG" : "FIPS140_2", 
+        "PLATFORM_TARGETS" : [
+            { "ppc64le_linux" : "defaultFipsTestTargets" },
+            { "s390x_linux" : "defaultFipsTestTargets" },
+            { "x86-64_linux" : "defaultFipsTestTargets" }
+        ]
+    },
+    {
+        "TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS",
+        "PLATFORM_TARGETS" : [
+            { "ppc64_aix" : "defaultFipsTestTargets" },
+            { "ppc64le_linux" : "defaultFipsTestTargets" },
+            { "s390x_linux" : "defaultFipsTestTargets" },
+            { "x86-64_linux" : "defaultFipsTestTargets" },
+            { "x86-64_windows" : "defaultFipsTestTargets" }
+        ]
+    },
+    {
+        "TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS.FIPS140-3",
+        "PLATFORM_TARGETS" : [
+            { "ppc64_aix" : "defaultFipsTestTargets" },
+            { "ppc64le_linux" : "defaultFipsTestTargets" },
+            { "s390x_linux" : "defaultFipsTestTargets" },
+            { "x86-64_linux" : "defaultFipsTestTargets" },
+            { "x86-64_windows" : "defaultFipsTestTargets" }
+        ]
+    }
+]
diff --git a/buildenv/jenkins/config/openj9/jdk8.json b/buildenv/jenkins/config/openj9/jdk8.json
@@ -0,0 +1,24 @@
+[
+    {
+        "TEST_FLAG" : "NONE", 
+        "PLATFORM_TARGETS" : [
+            { "aarch64_linux" : "defaultTestTargets" },
+            { "aarch64_mac" : "defaultTestTargets" },
+            { "ppc64_aix" : "defaultTestTargets" },
+            { "ppc64le_linux" : "defaultTestTargets" },
+            { "s390x_linux" : "defaultTestTargets" },
+            { "x86-64_linux" : "defaultTestTargets" },
+            { "x86-64_mac" : "defaultTestTargets" },
+            { "x86-64_windows" : "defaultTestTargets" },
+            { "x86-32_windows" : "defaultTestTargets" }
+        ]
+    },
+    {
+        "TEST_FLAG" : "FIPS140_2", 
+        "PLATFORM_TARGETS" : [
+            { "ppc64le_linux" : "defaultFipsTestTargets" },
+            { "s390x_linux" : "defaultFipsTestTargets" },
+            { "x86-64_linux" : "defaultFipsTestTargets" }
+        ]
+    }
+]