Skip to content

Commit

Permalink
Merge pull request #1185 from opendevstack/experimental
Browse files Browse the repository at this point in the history
Show Jenkins master/agent errors in the boxes, when they do not work as expected.
  • Loading branch information
victorpablosceruelo authored Sep 28, 2022
2 parents d2ad20b + b0229a6 commit 2063d3b
Show file tree
Hide file tree
Showing 14 changed files with 306 additions and 32 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
- Fix CI/CD problems in Jenkins pipelines ([#1177](https://github.com/opendevstack/ods-core/pull/1177))
- Fixes Python agent does not seems to have java in the path ([#685](https://github.com/opendevstack/ods-quickstarters/issues/685))
- Removes existing differences between jenkins agent base image in Centos 7 and UBI 8 ([#1181](https://github.com/opendevstack/ods-core/pull/1181))
- Upgrade to Java 11 Jenkins master and agents and increase logging to know why they sometimes die ([#1185](https://github.com/opendevstack/ods-core/pull/1185))

### Added

Expand Down
14 changes: 4 additions & 10 deletions jenkins/agent-base/Dockerfile.centos7
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,10 @@ ARG APP_DNS
ARG SNYK_DISTRIBUTION_URL
ARG AQUASEC_SCANNERCLI_URL

COPY ensure_java_jre_is_adequate.sh /usr/local/bin/
RUN rm -fv /etc/yum.repos.d/CentOS-Media.repo /etc/yum.repos.d/origin-local-release.repo \
&& ensure_java_jre_is_adequate.sh \
&& yum -y install openssl \
&& yum -y install java-11-openjdk-devel \
&& yum list installed | grep java \
&& yum -y remove java-1.8* \
&& yum -y update \
&& yum clean all \
&& rm -rf /var/cache/yum/*
Expand Down Expand Up @@ -115,14 +114,9 @@ RUN . /tmp/set_java_proxy.sh && echo $JAVA_OPTS
# It is setting the variable JAVA_TOOL_OPTIONS while it should not.
# Besides, we need to know if this variable has not been set.
# It is a problem very difficult to detect...
COPY fix_openshift_run_jnlp_client.sh /usr/local/bin/fix_openshift_run_jnlp_client.sh
RUN mv /usr/local/bin/run-jnlp-client /usr/local/bin/openshift-run-jnlp-client \
&& sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' /usr/local/bin/openshift-run-jnlp-client \
&& sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
/usr/local/bin/openshift-run-jnlp-client \
&& sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
/usr/local/bin/openshift-run-jnlp-client \
&& sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' /usr/local/bin/openshift-run-jnlp-client \
&& grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\)' /usr/local/bin/openshift-run-jnlp-client
&& fix_openshift_run_jnlp_client.sh /usr/local/bin/openshift-run-jnlp-client

COPY ods-run-jnlp-client.sh /usr/local/bin/run-jnlp-client

Expand Down
11 changes: 4 additions & 7 deletions jenkins/agent-base/Dockerfile.ubi8
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ ARG AQUASEC_SCANNERCLI_URL
# Add UBI repositories.
COPY yum.repos.d/ubi.repo /etc/yum.repos.d/ubi.repo

COPY ensure_java_jre_is_adequate.sh /usr/local/bin/
RUN cd /etc/yum.repos.d && rm -f localdev-* ci-rpm-mirrors.repo \
&& ensure_java_jre_is_adequate.sh \
&& yum -y install make glibc-langpack-en openssl \
&& yum -y update \
&& yum clean all \
Expand Down Expand Up @@ -115,14 +117,9 @@ COPY set_java_proxy.sh /tmp/set_java_proxy.sh
RUN . /tmp/set_java_proxy.sh && echo $JAVA_OPTS

# Customize entrypoint.
COPY fix_openshift_run_jnlp_client.sh /usr/local/bin/fix_openshift_run_jnlp_client.sh
RUN mv /usr/local/bin/run-jnlp-client /usr/local/bin/openshift-run-jnlp-client \
&& sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' /usr/local/bin/openshift-run-jnlp-client \
&& sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
/usr/local/bin/openshift-run-jnlp-client \
&& sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
/usr/local/bin/openshift-run-jnlp-client \
&& sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' /usr/local/bin/openshift-run-jnlp-client \
&& grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\)' /usr/local/bin/openshift-run-jnlp-client
&& fix_openshift_run_jnlp_client.sh /usr/local/bin/openshift-run-jnlp-client

COPY ods-run-jnlp-client.sh /usr/local/bin/run-jnlp-client

Expand Down
67 changes: 67 additions & 0 deletions jenkins/agent-base/ensure_java_jre_is_adequate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/bin/bash
set -eu -o pipefail

ME="$(basename $0)"
JAVA_INSTALLED_PKGS_LOGS="/tmp/java_installed_pkgs.log"
JAVA_11_INSTALLED_PKGS_LOGS="/tmp/java_11_installed_pkgs.log"
rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}

NEEDS_DEVEL=${1-""}
PKG_NAME_TAIL="headless"
if [ ! -z "${NEEDS_DEVEL}" ] && [ "" != "${NEEDS_DEVEL}" ]; then
NEEDS_DEVEL="true"
PKG_NAME_TAIL="devel"
else
NEEDS_DEVEL="false"
PKG_NAME_TAIL="headless"
fi

echo "${ME}: Needs development packages? ${NEEDS_DEVEL}"
echo " "
echo "${ME}: Listing versions of java installed: "
yum list installed | grep -i "\(java\|jre\)" | tee -a ${JAVA_INSTALLED_PKGS_LOGS}
touch ${JAVA_11_INSTALLED_PKGS_LOGS}
grep -i "java-11" ${JAVA_INSTALLED_PKGS_LOGS} > ${JAVA_11_INSTALLED_PKGS_LOGS} || echo "No java 11 packages found."

NEEDS_INSTALLATION="true"
if [ -f ${JAVA_11_INSTALLED_PKGS_LOGS} ]; then
if grep -qi "${PKG_NAME_TAIL}" ${JAVA_11_INSTALLED_PKGS_LOGS} ; then
NEEDS_INSTALLATION="false"
fi
fi

# We need devel package in masters to have jar binary.
if [ "true" == "${NEEDS_INSTALLATION}" ]; then
echo "${ME}:Java-11 is *not* installed. Installing..."
if [ "true" == "${NEEDS_DEVEL}" ]; then
yum -y install java-11-openjdk-devel
else
yum -y install java-11-openjdk-headless
fi
else
echo "${ME}: Java-11 is already installed."
fi

if grep -qi "java-1.8" ${JAVA_INSTALLED_PKGS_LOGS} ; then
echo "${ME}: Java-8 is installed. Removing..."
yum -y remove java-1.8*
else
echo "${ME}: Java-8 is not installed. Correct."
fi

rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}

echo " "
echo "${ME}: Checking java tool versions: "
if [ "true" == "${NEEDS_DEVEL}" ]; then
jar --version
fi

NO_JAVA_LINK="false"
java -version || NO_JAVA_LINK="true"
if [ "true" == "${NO_JAVA_LINK}" ]; then
JAVA_HOME_FOLDER=$(ls -lah /usr/lib/jvm | grep "java-11-openjdk-11.*\.x86_64" | awk '{print $NF}' | head -1)
JAVA_HOME="/usr/lib/jvm/${JAVA_HOME_FOLDER}"
alternatives --set java ${JAVA_HOME}/bin/java
fi
java -version
23 changes: 23 additions & 0 deletions jenkins/agent-base/fix_openshift_run_jnlp_client.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash
set -eu -o pipefail

FILEPATH=${1-"/usr/local/bin/openshift-run-jnlp-client"}

if [ ! -f ${FILEPATH} ]; then
echo " "
echo "ERROR: File does not exist: ${FILEPATH}"
echo " "
exit 1
fi

sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' ${FILEPATH}
sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
${FILEPATH}
sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
${FILEPATH}
sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' ${FILEPATH}
sed -i 's|curl\s*-sS\s*|curl -sSLv |g' ${FILEPATH}

grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\|curl\)' ${FILEPATH}


17 changes: 15 additions & 2 deletions jenkins/master/Dockerfile.centos7
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,20 @@ ENV TAILOR_VERSION=1.3.4
USER root

COPY ./import_certs.sh /usr/local/bin/import_certs.sh
RUN import_certs.sh
COPY ./ensure_java_jre_is_adequate.sh /usr/local/bin/ensure_java_jre_is_adequate.sh
COPY ./fix_openshift_scripts.sh /usr/local/bin/fix_openshift_scripts.sh
COPY ./clean_yum_cache.sh /usr/local/bin/clean_yum_cache.sh
RUN import_certs.sh \
&& rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key \
&& ensure_java_jre_is_adequate.sh master \
&& fix_openshift_scripts.sh \
&& yum -y update \
&& clean_yum_cache.sh


# To debug. Warning: too much logs if we enable it.
#RUN sed -i 's|\#\!\s*/bin/bash.*|\#\!/bin/bash -xeu|g' /usr/local/bin/install-plugins.sh \
# && head -n 10 /usr/local/bin/install-plugins.sh

# Copy configuration and plugins.
COPY plugins.centos7.txt /opt/openshift/configuration/plugins.txt
Expand All @@ -31,7 +44,7 @@ RUN chown :0 /etc/pki/java/cacerts && chmod ugo+w /etc/pki/java/cacerts

# Install Tailor.
RUN cd /tmp \
&& curl -LOv https://github.com/opendevstack/tailor/releases/download/v${TAILOR_VERSION}/tailor-linux-amd64 \
&& curl -sSLO https://github.com/opendevstack/tailor/releases/download/v${TAILOR_VERSION}/tailor-linux-amd64 \
&& mv tailor-linux-amd64 /usr/local/bin/tailor \
&& chmod a+x /usr/local/bin/tailor

Expand Down
9 changes: 8 additions & 1 deletion jenkins/master/Dockerfile.rhel7
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,14 @@ ENV TAILOR_VERSION=1.3.4
USER root

COPY ./import_certs.sh /usr/local/bin/import_certs.sh
RUN import_certs.sh
COPY ./ensure_java_jre_is_adequate.sh /usr/local/bin/ensure_java_jre_is_adequate.sh
COPY ./fix_openshift_scripts.sh /usr/local/bin/fix_openshift_scripts.sh
COPY ./clean_yum_cache.sh /usr/local/bin/clean_yum_cache.sh
RUN import_certs.sh \
&& rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key \
&& ensure_java_jre_is_adequate.sh master \
&& fix_openshift_scripts.sh \
&& clean_yum_cache.sh

# Copy configuration and plugins.
COPY plugins.rhel7.txt /opt/openshift/configuration/plugins.txt
Expand Down
12 changes: 11 additions & 1 deletion jenkins/master/Dockerfile.ubi8
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,17 @@ ENV TAILOR_VERSION=1.3.4
USER root

COPY ./import_certs.sh /usr/local/bin/import_certs.sh
RUN import_certs.sh
COPY ./ensure_java_jre_is_adequate.sh /usr/local/bin/ensure_java_jre_is_adequate.sh
COPY ./fix_openshift_scripts.sh /usr/local/bin/fix_openshift_scripts.sh
COPY ./clean_yum_cache.sh /usr/local/bin/clean_yum_cache.sh
RUN import_certs.sh \
&& rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key \
&& sed -i 's|enabled\s*=\s*1|enabled=0|g' /etc/yum.repos.d/ci-rpm-mirrors.repo \
&& sed -i 's|enabled\s*=\s*1|enabled=0|g' /etc/yum.repos.d/localdev-* \
&& sed -i 's|enabled\s*=\s*1|enabled=0|g' /etc/yum.repos.d/epel.repo \
&& ensure_java_jre_is_adequate.sh master \
&& fix_openshift_scripts.sh \
&& clean_yum_cache.sh

# Copy configuration and plugins.
COPY plugins.ubi8.txt /opt/openshift/configuration/plugins.txt
Expand Down
7 changes: 7 additions & 0 deletions jenkins/master/clean_yum_cache.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash
set -eu -o pipefail

ME="$(basename $0)"

yum clean all || true
rm -rf /var/cache/yum/* || true
67 changes: 67 additions & 0 deletions jenkins/master/ensure_java_jre_is_adequate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/bin/bash
set -eu -o pipefail

ME="$(basename $0)"
JAVA_INSTALLED_PKGS_LOGS="/tmp/java_installed_pkgs.log"
JAVA_11_INSTALLED_PKGS_LOGS="/tmp/java_11_installed_pkgs.log"
rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}

NEEDS_DEVEL=${1-""}
PKG_NAME_TAIL="headless"
if [ ! -z "${NEEDS_DEVEL}" ] && [ "" != "${NEEDS_DEVEL}" ]; then
NEEDS_DEVEL="true"
PKG_NAME_TAIL="devel"
else
NEEDS_DEVEL="false"
PKG_NAME_TAIL="headless"
fi

echo "${ME}: Needs development packages? ${NEEDS_DEVEL}"
echo " "
echo "${ME}: Listing versions of java installed: "
yum list installed | grep -i "\(java\|jre\)" | tee -a ${JAVA_INSTALLED_PKGS_LOGS}
touch ${JAVA_11_INSTALLED_PKGS_LOGS}
grep -i "java-11" ${JAVA_INSTALLED_PKGS_LOGS} > ${JAVA_11_INSTALLED_PKGS_LOGS} || echo "No java 11 packages found."

NEEDS_INSTALLATION="true"
if [ -f ${JAVA_11_INSTALLED_PKGS_LOGS} ]; then
if grep -qi "${PKG_NAME_TAIL}" ${JAVA_11_INSTALLED_PKGS_LOGS} ; then
NEEDS_INSTALLATION="false"
fi
fi

# We need devel package in masters to have jar binary.
if [ "true" == "${NEEDS_INSTALLATION}" ]; then
echo "${ME}:Java-11 is *not* installed. Installing..."
if [ "true" == "${NEEDS_DEVEL}" ]; then
yum -y install java-11-openjdk-devel
else
yum -y install java-11-openjdk-headless
fi
else
echo "${ME}: Java-11 is already installed."
fi

if grep -qi "java-1.8" ${JAVA_INSTALLED_PKGS_LOGS} ; then
echo "${ME}: Java-8 is installed. Removing..."
yum -y remove java-1.8*
else
echo "${ME}: Java-8 is not installed. Correct."
fi

rm -fv ${JAVA_INSTALLED_PKGS_LOGS} ${JAVA_11_INSTALLED_PKGS_LOGS}

echo " "
echo "${ME}: Checking java tool versions: "
if [ "true" == "${NEEDS_DEVEL}" ]; then
jar --version
fi

NO_JAVA_LINK="false"
java -version || NO_JAVA_LINK="true"
if [ "true" == "${NO_JAVA_LINK}" ]; then
JAVA_HOME_FOLDER=$(ls -lah /usr/lib/jvm | grep "java-11-openjdk-11.*\.x86_64" | awk '{print $NF}' | head -1)
JAVA_HOME="/usr/lib/jvm/${JAVA_HOME_FOLDER}"
alternatives --set java ${JAVA_HOME}/bin/java
fi
java -version
25 changes: 25 additions & 0 deletions jenkins/master/fix_openshift_scripts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/bin/bash
set -eu -o pipefail

ME="$(basename $0)"
echo "${ME}: INFO: Fixing openshift scripts..."

FILE_TO_MODIFY=${1-"/usr/libexec/s2i/run"}

if [ -f ${FILE_TO_MODIFY} ]; then
sed -i 's|\#\!/bin/bash|\#\!/bin/bash -x|g' ${FILE_TO_MODIFY}
sed -i "s|^\s*JAVA_TOOL_OPTIONS\s*=.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
${FILE_TO_MODIFY}
sed -i "s|^\s*export\s*JAVA_TOOL_OPTIONS.*| echo 'WARNING: JAVA_TOOL_OPTIONS env variable is UNSET.'|g" \
${FILE_TO_MODIFY}
sed -i 's|^\(\s*\)JAVA_GC_OPTS\s*=.*|\1JAVA_GC_OPTS=|g' ${FILE_TO_MODIFY}
grep -B 3 -A 3 -i '\(bash\|JAVA_TOOL_OPTIONS\|JAVA_GC_OPTS\)' ${FILE_TO_MODIFY}
else
echo " "
echo "${ME}: WARNING: Could not modify file because it does not exist: ${FILE_TO_MODIFY} "
echo " "
echo " "
fi

echo "${ME}: INFO: Fixed openshift scripts."

45 changes: 45 additions & 0 deletions jenkins/master/install_jenkins_lts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/bin/bash
set -eu -o pipefail

ME="$(basename $0)"
echo "${ME}: Upgrading Jenkins to latest LTS version available..."

# sudo wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo
# sudo rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key
# sudo yum upgrade
# Add required dependencies for the jenkins package
# sudo yum install java-11-openjdk
# sudo yum install jenkins
# sudo systemctl daemon-reload

DEFAULT_TARGET="/usr/lib/jenkins/jenkins.war"
TARGET="${DEFAULT_TARGET}"

curl -sSLO https://get.jenkins.io/war-stable/latest/jenkins.war

if [ ! -f "${TARGET}" ]; then
echo "${ME}: File does not exist: ${TARGET}"
TARGET="$(find /usr/ -name jenkins.war)"
echo "${ME}: New target: ${TARGET}"
fi

if [ -f "${TARGET}" ]; then
echo "${ME}: Upgrading Jenkins to latest LTS version... "
rm -fv ${TARGET}
mv -vf jenkins.war ${TARGET}
ls -lah ${TARGET}
else
echo "${ME}: ERROR: Cannot upgrade Jenkins version."
exit 1
fi

if [ ! -f "${DEFAULT_TARGET}" ]; then
DEFAULT_TARGET_FOLDER="$(dirname ${DEFAULT_TARGET})"
if [ ! -d ${DEFAULT_TARGET_FOLDER} ]; then
mkdir -pv ${DEFAULT_TARGET_FOLDER}
fi
cd ${DEFAULT_TARGET_FOLDER} && ln -sv ${TARGET} .
fi
ls -la ${DEFAULT_TARGET} ${TARGET}
echo "${ME}: INFO: Jenkins was upgraded to latest LTS version."

Loading

0 comments on commit 2063d3b

Please sign in to comment.