Skip to content

Commit

Permalink
Set runtime shell requirements within Jenkins Pipeline (#3171)
Browse files Browse the repository at this point in the history 
This PR updates the Jenkins Pipeline script to source `gw_setup.sh` from
within the PR itself for when ever a shell command is issued that needs
it.

Currently the runtime bash shell environment requirements for running
the **global-workflow** (set by sourcing
`$HOMEgfs/workflow/gw_setup.sh`) is established by the user's shell that
makes the connection from the Java remote agent to the Jenkins
Controller. This update simply sources `gw_setup.sh` directly during run
time when ever needed instead.

**NOTE:** This does not apply for **GitHub CLI** and **pyGitHub** since
they are not currently supported by **Spack Stack** and therefore not
incorporated in to `gw_setup.sh`. These remaining dependances are the
responsibility of the _role account_ holders that manage the Jenkins
Controller to Node Agent connections.

---------

Co-authored-by: Terry McGuinness <[email protected]>
Co-authored-by: Rahul Mahajan <[email protected]>
  • Loading branch information
@TerrenceMcGuinness-NOAA @aerorahul
3 people authored Jan 3, 2025
1 parent da9d717 commit e5d857b
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 16 deletions.
45 changes: 36 additions & 9 deletions ci/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,14 @@ pipeline {
}
}
try {
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}
""")
gist_url=sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}
""", returnStdout: true).trim()
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
Expand All @@ -158,7 +164,10 @@ pipeline {
}
}
// Get a list of CI cases to run
CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
CI_CASES = sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}
""", returnStdout: true).trim().split()
echo "Cases to run: ${CI_CASES}"
}
}
Expand All @@ -179,7 +188,10 @@ pipeline {
script {
env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
try {
error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
error_output = sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml
""", returnStdout: true).trim()
} catch (Exception error_create) {
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
error("Case ${caseName} failed to create experiment directory")
Expand All @@ -194,10 +206,19 @@ pipeline {
def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
sh(script: " rm -f ${error_file}")
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'")
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'
""")
sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}
""")
} catch (Exception error_experment) {
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}
""")
ws(CUSTOM_WORKSPACE) {
def error_logs = ""
def error_logs_message = ""
Expand All @@ -217,9 +238,15 @@ pipeline {
}
}
try {
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
gist_url = sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}
""", returnStdout: true).trim()
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
sh(script: """
source ${HOMEgfs}/workflow/gw_setup.sh
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}
""")
} catch (Exception error_comment) {
echo "Failed to comment on PR: ${error_comment.getMessage()}"
}
Expand Down
15 changes: 8 additions & 7 deletions ci/scripts/utils/launch_java_agent.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,14 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov"
controller_user=${controller_user:-"terry.mcguinness"}
controller_user_auth_token="jenkins_token"

HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
host=$(hostname)

#########################################################################
# Set up runtime environment varibles for accounts on supproted machines
#########################################################################

source "${HOMEgfs}/ush/detect_machine.sh"
source "${HOMEGFS_}/ush/detect_machine.sh"
case ${MACHINE_ID} in
hera | orion | hercules | wcoss2 | gaea)
echo "Launch Jenkins Java Controler on ${MACHINE_ID}";;
Expand All @@ -84,10 +84,10 @@ esac
LOG=lanuched_agent-$(date +%Y%m%d%M).log
rm -f "${LOG}"

source "${HOMEgfs}/ush/module-setup.sh"
module use "${HOMEgfs}/modulefiles"
source "${HOMEGFS_}/ush/module-setup.sh"
module use "${HOMEGFS_}/modulefiles"
module load "module_gwsetup.${MACHINE_ID}"
source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}"

JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10"
if [[ ! -d "${JAVA_HOME}" ]]; then
Expand All @@ -102,9 +102,10 @@ JAVA="${JAVA_HOME}/bin/java"
echo "JAVA VERSION: "
${JAVA} -version

export GH="${HOME}/bin/gh"
[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin"
GH=$(command -v gh || echo "${HOME}/bin/gh")
[[ -f "${GH}" ]] || ( echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 )
${GH} --version
export GH

check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
if [[ "${check_mark}" != "✓" ]]; then
Expand Down

0 comments on commit e5d857b

Please sign in to comment.