From 80b6a9ae8cadba21c1c77338895985e0627bf79b Mon Sep 17 00:00:00 2001 From: kapil-powar Date: Mon, 4 Dec 2023 18:20:28 +0530 Subject: [PATCH] Update Exclude files Update Exclude files - Update script to include latest exclude files from artifactory for JDK versions Signed-off-by: Kapil Powar --- buildenv/jenkins/jck_sync | 40 +++-- jck/jck-semiauto-updater/jckupdater.sh | 217 +++++++++++++++---------- 2 files changed, 154 insertions(+), 103 deletions(-) diff --git a/buildenv/jenkins/jck_sync b/buildenv/jenkins/jck_sync index 4cb806c572..5c3da25d9e 100644 --- a/buildenv/jenkins/jck_sync +++ b/buildenv/jenkins/jck_sync @@ -8,6 +8,9 @@ timestamps{ echo ("clone the SCM GIT repo") try { def gitConfig = scm.getUserRemoteConfigs().get(0) + timeout(time: 1, unit: 'HOURS') { + cleanWs disableDeferredWipeout: true, deleteDirs: true + } checkout scm: [$class: 'GitSCM', branches: [[name: "${scm.branches[0].name}"]], extensions: [ @@ -37,10 +40,11 @@ timestamps{ def syncJckMaterial(){ echo ("Starting script to sync JCK materials .....") - def JCK_VERSION = params.JDK_VERSION ? "-j ${params.JDK_VERSION}": "" - def JCK_GIT_BRANCH = params.JCK_GIT_BRANCH ? "-gb ${params.JCK_GIT_BRANCH}": "" - def ARTIFACTORY_URL = params.ARTIFACTORY_URL ? "-au ${params.ARTIFACTORY_URL}": "" - def JCK_GIT_REPO = params.JCK_GIT_REPO ? "-repo ${params.JCK_GIT_REPO}": "" + def JCK_VERSION_OPTION = params.JDK_VERSION ? "-j ${params.JDK_VERSION}": "" + def JCK_GIT_BRANCH_OPTION = params.JCK_GIT_BRANCH ? "-gb ${params.JCK_GIT_BRANCH}": "" + def ARTIFACTORY_URL_OPTION = params.ARTIFACTORY_URL ? "-au ${params.ARTIFACTORY_URL}": "" + def ARTIFACTORY_API_URL_OPTION = params.ARTIFACTORY_API_URL ? "-ap ${params.ARTIFACTORY_API_URL}": "" + def JCK_GIT_REPO_OPTION = params.JCK_GIT_REPO ? "-repo ${params.JCK_GIT_REPO}": "" def PR_NUMBER = 0 stage('JCKUpdater') { getJavaSDK() @@ -52,17 +56,17 @@ def syncJckMaterial(){ ]) { dir("${WORKSPACE}/aqa-tests/jck/jck-semiauto-updater"){ sshagent(credentials:["${params.USER_CREDENTIALS_ID}"], ignoreMissing: true){ - def SCRIPT = "./jckupdater.sh ${JCK_VERSION} -at \"\$ARTIFACTORY_CREDENTIALS\" ${JCK_GIT_REPO} -gt \"\$GIT_CREDENTIALS\" ${ARTIFACTORY_URL} ${JCK_GIT_BRANCH} " + def SCRIPT = "./jckupdater.sh ${JCK_VERSION_OPTION} -at \"\$ARTIFACTORY_CREDENTIALS\" ${JCK_GIT_REPO_OPTION} -gt \"\$GIT_CREDENTIALS\" ${ARTIFACTORY_URL_OPTION} ${ARTIFACTORY_API_URL_OPTION} ${JCK_GIT_BRANCH_OPTION} " def tempOutputFile = "temp_output.txt" def EXITCODE = sh(script: "${SCRIPT} > ${tempOutputFile}", returnStatus: true) + def scriptOutput = readFile(tempOutputFile).trim() + echo ("JCKUpdater script output:") + echo scriptOutput if (EXITCODE == 2) { error ("JCKUpdater script returned exit code 2, as no new update available. The job is completed without further action.") currentBuild.result = "SUCCESS" } else if (EXITCODE == 0) { echo ("Script returned exit code 0, new update is available and PR is created.") - def scriptOutput = readFile(tempOutputFile).trim() - echo ("JCKUpdater script output:") - echo scriptOutput def matcher = (scriptOutput =~ /PR_NUMBER=\s*(\d+)/) PR_NUMBER = matcher ? matcher[0][1] : null if (PR_NUMBER == null || PR_NUMBER == 0) { @@ -79,8 +83,8 @@ def syncJckMaterial(){ } stage('Test') { def CHILD_JOB_NAME="Grinder_JCK" - def BUILD_URL="${JENKINS_URL}job/${CHILD_JOB_NAME}/" - def CHILD_PARAMS = [] + def buildURL="${JENKINS_URL}job/${CHILD_JOB_NAME}/" + def childParams = [] // loop through all the params and change the parameters if needed params.each { param -> def value = param.value.toString() @@ -88,28 +92,28 @@ def syncJckMaterial(){ if (value.trim() == ""){ value = "autoBranch" //default branch name given in JCKUpdater.sh script } - CHILD_PARAMS << string(name: param.key, value: value) + childParams << string(name: param.key, value: value) } else { if (value == "true" || value == "false") { - CHILD_PARAMS << booleanParam(name: param.key, value: value.toBoolean()) + childParams << booleanParam(name: param.key, value: value.toBoolean()) } else { - CHILD_PARAMS << string(name: param.key, value: value) + childParams << string(name: param.key, value: value) } } } - def BUILD_INFO = build job: CHILD_JOB_NAME, parameters: CHILD_PARAMS - def COMMENT="Job [#${BUILD_INFO.number}](${BUILD_URL}${BUILD_INFO.number}) completed with status: ${BUILD_INFO.result}" - createComment(COMMENT,PR_NUMBER) + def buildInfo = build job: CHILD_JOB_NAME, parameters: childParams + def comment="Job [#${buildInfo.number}](${buildURL}${buildInfo.number}) completed with status: ${buildInfo.result}" + createComment(comment,PR_NUMBER) } } //Download JAVA SDK def getJavaSDK(){ def JDK_VERSION_OPTION = params.JDK_VERSION ? "-j ${params.JDK_VERSION}" : "" - def CUSTOMIZED_SDK_URL_OPTION = "-c ${params.CUSTOMIZED_SDK_URL}" + def CUSTOMIZED_SDK_OPTION = "-c ${params.CUSTOMIZED_SDK}" def CLONE_OPENJ9_OPTION = "--clone_openj9 false" def PLATFORM_OPTION = "-p x86-64_linux" - def GET_SH_CMD = "./get.sh -s `pwd`/.. ${PLATFORM_OPTION} ${JDK_VERSION_OPTION} ${CUSTOMIZED_SDK_URL_OPTION} ${CLONE_OPENJ9_OPTION} " + def GET_SH_CMD = "./get.sh -s `pwd`/.. ${PLATFORM_OPTION} ${JDK_VERSION_OPTION} ${CUSTOMIZED_SDK_OPTION} ${CLONE_OPENJ9_OPTION} " dir("${WORKSPACE}/aqa-tests") { if (params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID) { withCredentials([usernamePassword(credentialsId: "${params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { diff --git a/jck/jck-semiauto-updater/jckupdater.sh b/jck/jck-semiauto-updater/jckupdater.sh index f93a4ebd84..b2a9dd00d7 100755 --- a/jck/jck-semiauto-updater/jckupdater.sh +++ b/jck/jck-semiauto-updater/jckupdater.sh @@ -9,6 +9,8 @@ JCK_VERSION="" JCK_UPDATE_NUMBER="" JCK_GIT_REPO="" ARTIFACTORY_DOWNLOAD_URL="" +ARTIFACTORY_API_URL="" +ARTIFACTORY_EXC_URL="" ARTIFACTORY_TOKEN="" JCK_REPO_NAME="" GIT_USER="" @@ -21,6 +23,7 @@ usage () echo 'Usage : jckupdater.sh [--jck_version|-j ] : Indicate JCK version to update.' echo ' [--artifactory_token|-at] : Token to access JCK artifactory: https://eu.artifactory.swg-devops.com/artifactory/jim-jck-generic-local.' echo ' [--artifactory_url|-au] : Artifactory server URL to download JCK material' + echo ' [--artifactory_api_url|-ap] : Artifactory API URL to fetch details on JCK material' echo ' [--jck-repo|-repo] : JCK GIT repo to update.' echo ' [--git_token|-gt] : Git API Token to create PR.' echo ' [--jck_git_branch|-gb ] : Optional. JCK_GIT_BRANCH name to clone repo. Default is autoBranch' @@ -56,6 +59,9 @@ parseCommandLineArgs() "--artifactory_url" | "-au") ARTIFACTORY_DOWNLOAD_URL="$1"; shift;; + "--artifactory_api_url" | "-ap") + ARTIFACTORY_API_URL="$1"; shift;; + "--help" | "-h" ) usage; exit 0;; @@ -95,12 +101,10 @@ setup(){ JCK_FOLDER_SUFFIX="$JCK_VERSION"a fi echo "JCK_FOLDER_SUFFIX=" $JCK_FOLDER_SUFFIX - #check if given ARTIFACTORY_DOWNLOAD_URL is complete till JCK/tck to download material - if ! echo "$ARTIFACTORY_DOWNLOAD_URL" | grep -q "${JCK}/tck/"; then - ARTIFACTORY_DOWNLOAD_URL="${ARTIFACTORY_DOWNLOAD_URL}/${JCK}/tck/" - fi - - echo "ARTIFACTORY_DOWNLOAD_URL=$ARTIFACTORY_DOWNLOAD_URL" + ARTIFACTORY_JCK_DOWNLOAD_URL="${ARTIFACTORY_DOWNLOAD_URL}/${JCK}/" + ARTIFACTORY_EXC_URL="${ARTIFACTORY_API_URL}/${JCK}/exc/" + echo "ARTIFACTORY_JCK_DOWNLOAD_URL=$ARTIFACTORY_JCK_DOWNLOAD_URL" + echo "ARTIFACTORY_EXC_URL=$ARTIFACTORY_EXC_URL" echo "" mkdir $WORKSPACE @@ -109,39 +113,9 @@ setup(){ mkdir $WORKSPACE/jckmaterial } - -executeCmdWithRetry() -{ - set +e - count=0 - rt_code=-1 - # when the command is not found (code 127), do not retry - while [ "$rt_code" != 0 ] && [ "$rt_code" != 127 ] && [ "$count" -le 5 ] - do - if [ "$count" -gt 0 ]; then - sleep_time=3 - echo "error code: $rt_code. Sleep $sleep_time secs, then retry $count..." - sleep $sleep_time - - echo "check for $1. If found, the file will be removed." - if [ "$1" != "" ] && [ -f "$1" ]; then - echo "remove $1 before retry..." - rm $1 - fi - fi - echo "$2" - eval "$2" - rt_code=$? - count=$(( $count + 1 )) - done - set -e - return "$rt_code" -} - - - list() { - file_list=$(curl -ks -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} "${ARTIFACTORY_DOWNLOAD_URL}") + url="$1" + file_list=$(curl -ks -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} "${url}") # Use grep to filter out the content within tags file_names=$(echo "$file_list" | grep -o '' | sed 's///') @@ -150,11 +124,11 @@ list() { isLatestUpdate() { cd $WORKSPACE/jckmaterial - list # get the JCK update number from artifactory + list "${ARTIFACTORY_JCK_DOWNLOAD_URL}tck/" # get the JCK update number from artifactory last_file="" - for file in $file_names; do - last_file="$file" - done + for file in $file_names; do + last_file="$file" + done JCK_UPDATE_NUMBER=$last_file @@ -167,17 +141,85 @@ isLatestUpdate() { curl -o "build.txt" $GIT_URL -H "Authorization: token $GIT_TOKEN" echo -e "JCK version in build.txt:\n$(cat build.txt)\n\n" - if grep -q "$JCK_WITHOUT_BACKSLASH" build.txt; then + #check if latest exclude file is available + getExcludeFiles + exclude_files=$? + + result=$(grep -q "$JCK_WITHOUT_BACKSLASH" build.txt; echo $?) + + if [[ $result -eq 0 && $exclude_files -eq 0 ]]; then echo " JCK$JCK_VERSION material is $JCK_WITHOUT_BACKSLASH in the repo $GIT_URL. It is up to date. No need to pull changes" cleanup exit 2 - else - echo " JCK$JCK_VERSION $JCK_WITHOUT_BACKSLASH is latest and not in the repo $GIT_URL... Please proceed with download" - get_JAVA_SDK + else + if [ $result -ne 0 ]; then + echo " JCK$JCK_VERSION $JCK_WITHOUT_BACKSLASH is latest and not in the repo $GIT_URL... Please proceed with download" + getJavaSDK + else + echo " Latest exclude files are available for JCK$JCK_VERSION" + fi getJCKSources fi } +getLastModifiedForFile() { + file="$1" + curl -s -H "X-JFrog-Art-Api:$ARTIFACTORY_TOKEN" "$ARTIFACTORY_EXC_URL$file" | awk -F'"' '/lastModified/ {print $4}' +} + +getExcludeFiles() { + github_last_modified=$(curl -s -H "Authorization: token $GIT_TOKEN" "https://api.github.ibm.com/repos/runtimes/JCK$JCK_VERSION-unzipped/commits?path=excludes/jck$JCK_FOLDER_SUFFIX.jtx" | awk -F'"date":' '{if(NF>1){gsub(/[",]/,"",$2); print $2; exit}}') + #check if file is present in github. + if [[ -n "${github_last_modified}" ]]; then + github_date_only=$(date -u -d "${github_last_modified}" "+%Y-%m-%d") + fi + artifactory_file_list=$(curl -s -H "X-JFrog-Art-Api:$ARTIFACTORY_TOKEN" "$ARTIFACTORY_EXC_URL" -s | \ + awk -F'"uri" : "' '/"uri" : "\// { gsub(/".*$/, "", $2); print $2 }' | \ + awk '{gsub(/^\//, ""); print}' | \ + grep -v '/$' ) + + #if no exclude file is present + if [[ -z "$artifactory_file_list" ]]; then + echo "No exclude files present in Artifactory to update" + return 0 + else + declare -a last_modified_dates + declare -a file_last_modified + + echo "List of exclude files in Artifactory:" + echo "$artifactory_file_list" + + #store all files one by one + files=() + while IFS= read -r line; do + files+=("$line") + done <<< "$artifactory_file_list" + + for file in "${files[@]}"; do + artifactory_date=$(date -d "$(getLastModifiedForFile "$file")" "+%Y-%-m-%d") + last_modified_dates+=("$artifactory_date") + file_last_modified["$artifactory_date"]=$file + done + + artifactory_latest_modified=$(printf "%s\n" "${last_modified_dates[@]}" | sort -rV | head -1) + latest_file_name="${file_last_modified[$artifactory_latest_modified]}" + formatted_date=$(date -d "$artifactory_latest_modified" "+%Y-%m-%d") # reconvert month to value with leading zero for comparison + echo "Latest exclude file: $latest_file_name" + echo "Last Modified Date in GitHub: ${github_date_only}" + echo "Last Modified Date in Artifactory: ${formatted_date}" + + # Compare and find the latest last modified date only if file is present in github. Else update exclude file + if [[ -n "${github_last_modified}" && "${github_date_only}" > "${formatted_date}" ]]; then + echo "No need to update exclude file in GitHub" + return 0 + else + echo "Update exclude file with $latest_file_name from Artifactory" + return 1 + fi + fi + +} + ## Download directly from given URL under current folder getJCKSources() { cd $WORKSPACE/jckmaterial @@ -185,32 +227,38 @@ getJCKSources() { rm -rf build.txt echo "download jck materials..." - ARTIFACTORY_DOWNLOAD_URL=$ARTIFACTORY_DOWNLOAD_URL$JCK_UPDATE_NUMBER - echo $ARTIFACTORY_DOWNLOAD_URL + #download latest exclude file + if [ $exclude_files -eq 1 ]; then + curl -OLJSk -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} ${ARTIFACTORY_JCK_DOWNLOAD_URL}exc/$latest_file_name && unzip -o "$latest_file_name" + fi - list #get list of files to download - - IFS=$'\n' read -r -d '' -a file_names_array <<< "$file_names" + if [ $result -ne 0 ]; then + ARTIFACTORY_JCK_UPDATE_NUMBER_URL=${ARTIFACTORY_JCK_DOWNLOAD_URL}tck/$JCK_UPDATE_NUMBER + echo $ARTIFACTORY_JCK_UPDATE_NUMBER_URL - if [ "${ARTIFACTORY_DOWNLOAD_URL}" != "" ]; then - for file in "${file_names_array[@]:1}"; do - url="$ARTIFACTORY_DOWNLOAD_URL$file" - executeCmdWithRetry "${file##*/}" "_ENCODE_FILE_NEW=UNTAGGED curl -OLJSk -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} $url" + list "$ARTIFACTORY_JCK_UPDATE_NUMBER_URL" #get list of files to download - rt_code=$? - if [ $rt_code != 0 ]; then - echo "curl error code: $rt_code" - echo "Failed to retrieve $file. This is what we received of the file and MD5 sum:" - ls -ld $file + IFS=$'\n' read -r -d '' -a file_names_array <<< "$file_names" - exit 1 - fi - done + if [ "${ARTIFACTORY_JCK_UPDATE_NUMBER_URL}" != "" ]; then + for file in "${file_names_array[@]:1}"; do + url="$ARTIFACTORY_JCK_UPDATE_NUMBER_URL$file" + _ENCODE_FILE_NEW=UNTAGGED curl -OLJSk -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} $url + rt_code=$? + if [ $rt_code != 0 ]; then + echo "curl error code: $rt_code" + echo "Failed to retrieve $file. This is what we received of the file and MD5 sum:" + ls -ld $file + + exit 1 + fi + done + fi fi } #install Java -get_JAVA_SDK(){ +getJavaSDK() { if [[ $JAVA_HOME = "" ]] ; then cd $WORKSPACE/../../../../jdkbinary/j2sdk-image JAVA_SDK_PATH="$(pwd)" @@ -222,19 +270,16 @@ get_JAVA_SDK(){ #Unpack downloaded jar files extract() { cd $WORKSPACE/jckmaterial - - echo "install downloaded resources" + echo "install downloaded resources" for f in $WORKSPACE/jckmaterial/*.jar; do echo "Unpacking $f:" - #using default java on machine for local if [[ $JAVA_HOME != "" ]] ; then $JAVA_HOME/bin/java -jar $f -install shell_scripts -o $WORKSPACE/unpackjck else $JAVA_SDK_PATH/bin/java -jar $f -install shell_scripts -o $WORKSPACE/unpackjck fi - done cd $WORKSPACE/unpackjck ls -la @@ -242,8 +287,7 @@ extract() { } #Clone GIT branch. -gitClone() -{ +gitClone() { echo "Checking out git@github.ibm.com:$GIT_USER/$JCK_REPO_NAME and reset to git@github.ibm.com:runtimes/$JCK_REPO_NAME main branch in dir $GIT_REPO" cd $GIT_REPO git init @@ -258,27 +302,28 @@ gitClone() #Move unpacked files to GIT repository copyFilestoGITRepo() { - cd $GIT_REPO - rm -rf JCK-compiler-$JCK_FOLDER_SUFFIX - rm -rf JCK-runtime-$JCK_FOLDER_SUFFIX - rm -rf JCK-devtools-$JCK_FOLDER_SUFFIX - rm -rf headers - rm -rf natives - echo "copy unpacked JCK files from $WORKSPACE/unpackjck to local GIT dir $GIT_REPO" - - cd $WORKSPACE/unpackjck - for file in $WORKSPACE/unpackjck/*; do - echo $file - cp -rf $file $GIT_REPO - done + if [ $result -ne 0 ]; then + rm -rf JCK-compiler-$JCK_FOLDER_SUFFIX + rm -rf JCK-runtime-$JCK_FOLDER_SUFFIX + rm -rf JCK-devtools-$JCK_FOLDER_SUFFIX + rm -rf headers + rm -rf natives + echo "copy unpacked JCK files from $WORKSPACE/unpackjck to local GIT dir $GIT_REPO" + + cd $WORKSPACE/unpackjck + for file in $WORKSPACE/unpackjck/*; do + echo $file + cp -rf $file $GIT_REPO + done + fi #copy remaining file like .jtx, .kfl .html to GIT Repo for file in $WORKSPACE/jckmaterial/*; do if [[ "$file" != *.zip ]] && [[ "$file" != *.gz* ]] && [[ "$file" != *.jar ]];then if [[ "$file" == *.kfl* ]] || [[ "$file" == *.jtx* ]]; then echo "Copy $file to $GIT_REPO" - cp -rf $file $GIT_REPO/excludes + mkdir -p $GIT_REPO/excludes && cp -rf $file $GIT_REPO/excludes else cp -rf $file $GIT_REPO fi @@ -354,7 +399,9 @@ if [ "$JCK_VERSION" != "" ] && [ "$JCK_GIT_REPO" != "" ] && [ "$GIT_TOKEN" != "" cleanup setup isLatestUpdate - extract + if [ $result -ne 0 ]; then + extract + fi gitClone copyFilestoGITRepo checkChangesAndCommit