Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve AQA_Test_Pipeline for release testing #5512

Merged
merged 1 commit into from
Aug 28, 2024
Merged

Improve AQA_Test_Pipeline for release testing #5512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 127 additions & 32 deletions buildenv/jenkins/aqaTestPipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@ def TARGETS = params.TARGETS ?: "Grinder"
TARGETS = TARGETS.trim().split("\\s*,\\s*")

def PARALLEL = params.PARALLEL ? params.PARALLEL : "Dynamic"
def NUM_MACHINES = params.NUM_MACHINES ? params.NUM_MACHINES : 3

def NUM_MACHINES = ""
if (params.NUM_MACHINES) {
NUM_MACHINES = params.NUM_MACHINES
} else if (!params.TEST_TIME && PARALLEL == "Dynamic") {
// set default NUM_MACHINES to 3 if params.NUM_MACHINES and params.TEST_TIME are not set and PARALLEL is Dynamic
NUM_MACHINES = 3
karianna marked this conversation as resolved.
Show resolved Hide resolved
}
def SDK_RESOURCE = params.SDK_RESOURCE ? params.SDK_RESOURCE : "releases"
def TIME_LIMIT = params.TIME_LIMIT ? params.TIME_LIMIT : 10
def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN : false
def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN.toBoolean() : false
def TRSS_URL = params.TRSS_URL ? params.TRSS_URL : "https://trss.adoptium.net/"
def TEST_FLAG = (params.TEST_FLAG) ?: ""
def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
Expand All @@ -18,20 +25,62 @@ def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
def DEFAULT_SUFFIX = (env.BUILD_USER_ID) ? "${env.BUILD_USER_ID} - jdk-${params.JDK_VERSIONS}" : "jdk-${params.JDK_VERSIONS}"
def PIPELINE_DISPLAY_NAME = (params.PIPELINE_DISPLAY_NAME) ? "#${currentBuild.number} - ${params.PIPELINE_DISPLAY_NAME}" : "#${currentBuild.number} - ${DEFAULT_SUFFIX}"

def JOBS = [:]

// Set the AQA_TEST_PIPELINE Jenkins job displayName
currentBuild.setDisplayName(PIPELINE_DISPLAY_NAME)

def suffix = ""
if (TEST_FLAG) {
suffix = "_" + TEST_FLAG.toLowerCase().trim()
}
def defaultTestTargets = "sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf"
def defaultFipsTestTargets = "sanity.functional,extended.functional,sanity.openjdk,extended.openjdk,sanity.jck,extended.jck,special.jck"

JOBS = [:]
fail = false

def fail = false
int jobNum = 0
JDK_VERSIONS.each { JDK_VERSION ->
PLATFORMS.each { PLATFORM ->
if (params.PLATFORMS == "release") {
def configJson = []
node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
checkout scm
dir (env.WORKSPACE) {
def filePath = "./aqa-tests/buildenv/jenkins/config/${params.VARIANT}/"
filePath = filePath + "default.json"
if (fileExists(filePath + "jdk${JDK_VERSION}.json")) {
filePath = filePath + "jdk${JDK_VERSION}.json"
}
configJson = readJSON(file: filePath)
}
}

configJson.each { item ->
def releaseTestFlag = ""
releaseTestFlag = item.TEST_FLAG
item.PLATFORM_TARGETS.each { pt ->
pt.each{ p, t ->
def releasePlatform = p
def releaseTargets = ""
if (t.contains("defaultFipsTestTargets")) {
releaseTargets = t.replace("defaultFipsTestTargets","${defaultFipsTestTargets}")
} else {
releaseTargets = t.replace("defaultTestTargets","${defaultTestTargets}")
}
String[] releasePlatformArray = releasePlatform.split("\\s*,\\s*")
String[] releaseTargetsArray = releaseTargets.split("\\s*,\\s*")
generateJobs(JDK_VERSION, releaseTestFlag, releasePlatformArray, releaseTargetsArray)
}
}
}
} else {
generateJobs(JDK_VERSION, TEST_FLAG, PLATFORMS, TARGETS)
}
}
parallel JOBS
if (fail) {
currentBuild.result = "FAILURE"
}

def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) {
if (jobTestFlag == "NONE") {
jobTestFlag = ""
}
jobPlatforms.each { PLATFORM ->
String[] tokens = PLATFORM.split('_')
def os = tokens[1];
def arch = tokens[0];
Expand Down Expand Up @@ -63,30 +112,76 @@ JDK_VERSIONS.each { JDK_VERSION ->
if (params.VARIANT == "openj9") {
// get IBM Semeru CE
sdk_resource_value = "customized"
download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
}
}
echo "download_url: ${download_url}"

TARGETS.each { TARGET ->
jobTargets.each { TARGET ->
def TEST_JOB_NAME = "Grinder"
if (TARGET.contains("Grinder")) {
TEST_JOB_NAME = TARGET
} else {
TEST_JOB_NAME = "Test_openjdk${JDK_VERSION}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
def suffix = ""
if (jobTestFlag) {
suffix = "_" + jobTestFlag.toLowerCase().trim()
}
TEST_JOB_NAME = "Test_openjdk${jobJdkVersion}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
}
echo "TEST_JOB_NAME: ${TEST_JOB_NAME}"

def keep_reportdir = false
if (TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
if (TARGET.contains("functional") || TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
keep_reportdir = true
}
if (TARGET.contains("functional") || TARGET.contains("perf")) {
if (params.VARIANT == "temurin") {
PARALLEL = "None"

def DYNAMIC_COMPILE = false
if (!params.DYNAMIC_COMPILE) {
if (("${TARGET}".contains('functional')) || ("${TARGET}".contains('external'))) {
DYNAMIC_COMPILE = true
} else {
DYNAMIC_COMPILE = false
}
} else {
DYNAMIC_COMPILE = params.DYNAMIC_COMPILE ? params.DYNAMIC_COMPILE.toBoolean() : false
}

def VENDOR_TEST_REPOS = ''
def VENDOR_TEST_BRANCHES = ''
def VENDOR_TEST_DIRS = ''
int rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 0
if (params.VARIANT == "openj9") {
// default rerunIterations is 3 for openj9
rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 3
if (TARGET.contains('external')) {
PARALLEL = "None"
rerunIterations = 0
} else if (TARGET.contains('functional')) {
VENDOR_TEST_REPOS = '[email protected]:runtimes/test.git'
VENDOR_TEST_BRANCHES = params.ADOPTOPENJDK_BRANCH ?: 'master'
VENDOR_TEST_DIRS = 'functional'
} else if (TARGET.contains('jck')) {
VENDOR_TEST_REPOS = '[email protected]:runtimes/jck.git'
VENDOR_TEST_BRANCHES = "main"
VENDOR_TEST_DIRS = 'jck'
} else if (TARGET.contains('openjdk')) {
// only use osb repo for regular testing
if (TARGET.contains('special') && jobTestFlag == "") {
VENDOR_TEST_REPOS = '[email protected]:runtimes/osb-tests.git'
VENDOR_TEST_BRANCHES = "ibm_tlda"
VENDOR_TEST_DIRS = 'openjdk'
}
}

if (TARGET.contains("FIPS") || (TARGET.contains("dev"))) {
rerunIterations = 0
}
} else if (params.VARIANT == "temurin") {
if (TARGET.contains("functional") || TARGET.contains("perf")) {
PARALLEL = "None"
}
}
echo "AUTO_AQA_GEN: ${AUTO_AQA_GEN}"
// Grinder job has special settings and should be regenerated specifically, not via aqaTestPipeline
if (AUTO_AQA_GEN && !TEST_JOB_NAME.contains("Grinder")) {
String[] targetTokens = TARGET.split("\\.")
Expand All @@ -96,7 +191,7 @@ JDK_VERSIONS.each { JDK_VERSION ->
string(name: 'TEST_JOB_NAME', value: TEST_JOB_NAME),
string(name: 'LEVELS', value: level),
string(name: 'GROUPS', value: group),
string(name: 'JDK_VERSIONS', value: JDK_VERSION),
string(name: 'JDK_VERSIONS', value: jobJdkVersion),
string(name: 'ARCH_OS_LIST', value: PLATFORM),
string(name: 'JDK_IMPL', value: jdk_impl),
booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: LIGHT_WEIGHT_CHECKOUT)
Expand Down Expand Up @@ -124,8 +219,8 @@ JDK_VERSIONS.each { JDK_VERSION ->
childParams << booleanParam(name: param.key, value: LIGHT_WEIGHT_CHECKOUT.toBoolean())
} else if (param.key == "TIME_LIMIT") {
childParams << string(name: param.key, value: TIME_LIMIT.toString())
} else if (param.key == "TRSS_URL") {
childParams << string(name: param.key, value: TRSS_URL)
} else if (param.key == "TEST_FLAG") {
childParams << string(name: param.key, value: jobTestFlag)
} else if (param.key == "KEEP_REPORTDIR") {
childParams << booleanParam(name: param.key, value: keep_reportdir.toBoolean())
} else {
Expand All @@ -137,17 +232,22 @@ JDK_VERSIONS.each { JDK_VERSION ->
}
}
}
childParams << booleanParam(name: "DYNAMIC_COMPILE", value: DYNAMIC_COMPILE.toBoolean())
childParams << booleanParam(name: "GENERATE_JOBS", value: AUTO_AQA_GEN.toBoolean())
childParams << string(name: "JDK_VERSION", value: JDK_VERSION)
childParams << string(name: "PLATFORM", value: PLATFORM)
childParams << string(name: "JDK_IMPL", value: jdk_impl)
childParams << string(name: "JDK_VERSION", value: jobJdkVersion)
childParams << string(name: "PLATFORM", value: PLATFORM)
childParams << string(name: "RERUN_ITERATIONS", value: rerunIterations.toString())
childParams << string(name: "VENDOR_TEST_BRANCHES", value: VENDOR_TEST_BRANCHES)
childParams << string(name: "VENDOR_TEST_DIRS", value: VENDOR_TEST_DIRS)
childParams << string(name: "VENDOR_TEST_REPOS", value: VENDOR_TEST_REPOS)

jobNum++
int jobNum = JOBS.size() + 1
JOBS["${TEST_JOB_NAME}_${jobNum}"] = {
def downstreamJob = build job: TEST_JOB_NAME, parameters: childParams, propagate: false, wait: true
def downstreamJobResult = downstreamJob.getResult()
echo " ${TEST_JOB_NAME} result is ${downstreamJobResult}"
if (downstreamJob.getResult() == 'SUCCESS' || downstreamJob.getResult() == 'UNSTABLE') {
echo "${TEST_JOB_NAME} result is ${downstreamJobResult}"
if (downstreamJobResult == 'SUCCESS' || downstreamJobResult == 'UNSTABLE') {
echo "[NODE SHIFT] MOVING INTO CONTROLLER NODE..."
node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
cleanWs disableDeferredWipeout: true, deleteDirs: true
Expand Down Expand Up @@ -185,9 +285,4 @@ JDK_VERSIONS.each { JDK_VERSION ->
}
}
}
}
parallel JOBS
if (fail) {
currentBuild.result = "FAILURE"
}

}
43 changes: 43 additions & 0 deletions buildenv/jenkins/config/openj9/default.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
[
{
"TEST_FLAG" : "NONE",
"PLATFORM_TARGETS" : [
{ "aarch64_linux" : "defaultTestTargets,dev.external" },
{ "aarch64_mac" : "defaultTestTargets" },
{ "ppc64_aix" : "defaultTestTargets" },
{ "ppc64le_linux" : "defaultTestTargets,dev.external" },
{ "s390x_linux" : "defaultTestTargets,dev.external" },
{ "x86-64_linux" : "defaultTestTargets,dev.external" },
{ "x86-64_mac" : "defaultTestTargets" },
{ "x86-64_windows" : "defaultTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_2",
"PLATFORM_TARGETS" : [
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS",
"PLATFORM_TARGETS" : [
{ "ppc64_aix" : "defaultFipsTestTargets" },
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" },
{ "x86-64_windows" : "defaultFipsTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS.FIPS140-3",
"PLATFORM_TARGETS" : [
{ "ppc64_aix" : "defaultFipsTestTargets" },
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" },
{ "x86-64_windows" : "defaultFipsTestTargets" }
]
}
]
24 changes: 24 additions & 0 deletions buildenv/jenkins/config/openj9/jdk8.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
[
{
"TEST_FLAG" : "NONE",
"PLATFORM_TARGETS" : [
{ "aarch64_linux" : "defaultTestTargets" },
{ "aarch64_mac" : "defaultTestTargets" },
{ "ppc64_aix" : "defaultTestTargets" },
{ "ppc64le_linux" : "defaultTestTargets" },
{ "s390x_linux" : "defaultTestTargets" },
{ "x86-64_linux" : "defaultTestTargets" },
{ "x86-64_mac" : "defaultTestTargets" },
{ "x86-64_windows" : "defaultTestTargets" },
{ "x86-32_windows" : "defaultTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_2",
"PLATFORM_TARGETS" : [
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" }
]
}
]