Draft PR for review comments of infraclouds contributions

.build/build-jars.sh

@@ -19,7 +19,7 @@
 
 # variables, with defaults
 [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)"
-
+echo "$CASSANDRA_DIR from build_jar"
 # pre-conditions
 command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; }
 [ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; }

.build/docker/_docker_run.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -30,7 +30,7 @@
 [ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)"
 [ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build"
 [ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; }
-
+echo "$CASSANDRA_DIR from _docker_run"
 java_version_default=`grep 'property\s*name="java.default"' ${cassandra_dir}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'`
 java_version_supported=`grep 'property\s*name="java.supported"' ${cassandra_dir}/build.xml |sed -ne 's/.*value="\([^"]*\)".*/\1/p'`
 
@@ -111,9 +111,13 @@ container_name="cassandra_${dockerfile/.docker/}_${un_script_name}_jdk${java_ver
 #  change ant's build directory to $DIST_DIR
 #  set java to java_version
 #  execute the run_script
+# docker_command="export ANT_OPTS=\"-Dbuild.dir=\${DIST_DIR} ${CASSANDRA_DOCKER_ANT_OPTS}\" ; \
+#                 source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_version} ; \
+#                 \${CASSANDRA_DIR}/.build/${run_script} ${@:4} ; exit \$? "
+
 docker_command="export ANT_OPTS=\"-Dbuild.dir=\${DIST_DIR} ${CASSANDRA_DOCKER_ANT_OPTS}\" ; \
-                source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_version} ; \
-                \${CASSANDRA_DIR}/.build/${run_script} ${@:4} ; exit \$? "
+                source /home/build/cassandra/.build/docker/_set_java.sh ${java_version} ; \
+                /home/build/cassandra/.build/${run_script} ${@:4} ; exit \$? "
 
 # run without the default seccomp profile
 # re-use the host's maven repository
@@ -122,9 +126,11 @@ container_id=$(docker run --name ${container_name} -d --security-opt seccomp=unc
     ${docker_volume_opt} \
     ${image_name} sleep 1h)
 
+
 echo "Running container ${container_name} ${container_id}"
+#docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh build $(id -u) $(id -g)"
 
-docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh build $(id -u) $(id -g)"
+docker exec --user root ${container_name} bash -c "/home/build/cassandra/.build/docker/_create_user.sh build $(id -u) $(id -g)"
 docker exec --user build ${container_name} bash -c "${docker_command}"
 RETURN=$?
 

.build/docker/_set_java.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -e -x
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -24,7 +24,8 @@
 # variables, with defaults
 [ "x${CASSANDRA_DIR}" != "x" ] || { CASSANDRA_DIR="$(pwd)"; }
 [ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; }
-
+echo "$CASSANDRA_DIR from _set_java"
+echo $(pwd)
 # pre-conditions
 [ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; }
 

.build/docker/run-tests.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash +x
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -82,7 +82,6 @@ dockerfile="ubuntu2004_test.docker"
 image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)"
 image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
 docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/home/cassandra/cassandra/build -v ${HOME}/.m2/repository:/home/cassandra/.m2/repository"
-
 # Look for existing docker image, otherwise build
 if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
   # try docker login to increase dockerhub rate limits
@@ -196,7 +195,7 @@ logfile="${build_dir}/test/logs/docker_attach_${container_name}.log"
 #  set java to java_version
 #  execute the run_script
 docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_version} ; \
-            \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh ${target} ${split_chunk} ; exit \$?"
+            \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh ${target} ${split_chunk}; exit \$?"
 
 # start the container, timeout after 4 hours
 docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep 4h)
@@ -210,6 +209,7 @@ docker exec --user root ${container_name} update-alternatives --set python /usr/
 docker exec --user cassandra ${container_name} bash -c "${docker_command}" | tee -a ${logfile}
 status=$?
 
+
 if [ "$status" -ne 0 ] ; then
     echo "${docker_id} failed (${status}), debug…"
     docker inspect ${docker_id}
@@ -222,8 +222,8 @@ if [ "$status" -ne 0 ] ; then
     echo "–––"
     echo "Failure."
 fi
-# docker stop in background, ignore errors
-( nohup docker stop ${docker_id} >/dev/null 2>/dev/null & )
+docker stop in background, ignore errors
+#( nohup docker stop ${docker_id} >/dev/null 2>/dev/null & )
 
 xz -f ${logfile} 2>/dev/null
 

.build/run-ci.sh

@@ -0,0 +1,260 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# variables, with defaults
+[ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)"
+[ "x${KUBECONFIG}" != "x" ]    || KUBECONFIG="${HOME}/.kube/config"
+[ "x${KUBE_NS}" != "x" ]       || KUBE_NS="default" # FIXME – doesn't work in other namespaces :shrug:
+
+# pre-conditions
+command -v kubectl >/dev/null 2>&1 || { echo >&2 "kubectl needs to be installed"; exit 1; }
+command -v helm >/dev/null 2>&1 || { echo >&2 "helm needs to be installed"; exit 1; }
+
+# Parse command-line arguments
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        -c|--kubeconfig)
+            KUBECONFIG="$2"  # This sets the KUBECONFIG variable to the next argument.
+            shift            # This shifts the arguments to the left, discarding the current argument and moving to the next one.
+            shift            # This is an additional shift to move to the argument after the option value.
+            ;;
+        -ctx|--kubecontext)
+            unset KUBECONFIG
+            KUBECONTEXT="$2" # This sets the KUBECONTEXT variable to the next argument.
+            shift            # This shifts the arguments to the left, discarding the current argument and moving to the next one.
+            shift            # This is an additional shift to move to the argument after the option value.
+            ;;
+        --include-test-stage)
+            INCLUDE_TEST_STAGE="$2"
+            shift
+            shift
+            ;;
+        --repo-url)
+            REPO_URL="$2"
+            shift
+            shift;;
+        --repo-branch)
+            REPO_BRANCH="$2"
+            shift
+            shift
+            ;;
+        --targets)
+            TARGETS="$2"
+            shift
+            shift
+            ;;
+        --tear-down)
+            TEAR_DOWN="$2"
+            shift
+            shift
+            ;;
+        --help)
+            echo "Usage: your_script.sh [options]"
+            echo "Options:"
+            echo "  -c, --kubeconfig <file>       Specify the path to a Kubeconfig file."
+            echo "  -ctx, --kubecontext <context> Specify the Kubernetes context to use."
+            echo "  --include-test-stage <value>  Include test stage/s."
+            echo "  --repo-url <url>              Specify the repository URL."
+            echo "  --repo-branch <branch>        Specify the repository branch."
+            echo "  --targets <target>            Specify the build targets."
+            echo "  --tear-down <value>           Tear down Jenkins Instance and Jenkins Operator, true or false."
+            echo "  --help                        Show this help message."
+            exit 0 
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "$KUBECONFIG" ] && [ -z "$KUBECONTEXT" ]; then
+    echo "Please provide either the path to the kubeconfig using -c|--kubeconfig option or the kubecontext using -ctx|--kubecontext option."
+    exit 1
+fi
+
+# This sets the kubeconfig and kubecontext if provided
+if [ -n "$KUBECONFIG" ]; then
+    export KUBECONFIG="$KUBECONFIG"
+fi
+
+if [ -n "$KUBECONTEXT" ]; then
+    kubectl config use-context "$KUBECONTEXT"
+fi
+
+if ! kubectl get namespace ${KUBE_NS} >/dev/null 2>/dev/null ; then
+    kubectl create namespace ${KUBE_NS}
+fi
+
+if [ -n "$INCLUDE_TEST_STAGE" ]; then
+    # The variable is not empty, so it has a value
+    echo "INCLUDE_TEST_STAGE is not empty. Its value is: $INCLUDE_TEST_STAGE"
+else
+    # The variable is empty, so assign a default value
+    INCLUDE_TEST_STAGE="lint,stress,fqltool"
+    echo "INCLUDE_TEST_STAGE is empty. Assigning default value: $INCLUDE_TEST_STAGE"
+fi
+
+
+if [ -n "$TARGETS" ]; then
+    # The variable is not empty, so it has a value
+    echo "TARGETS is not empty. Its value is: $TARGETS"
+else
+    # The variable is empty, so assign a default value
+    TARGETS=".jenkins/job/DslJob.jenkins"
+    echo "TARGETS is empty. Assigning default value: $TARGETS"
+fi
+
+if [ -n "$REPO_URL" ]; then
+    # The variable is not empty, so it has a value
+    echo "REPO_URL is not empty. Its value is: $REPO_URL"
+else
+    # The variable is empty, so assign a default value
+    REPO_URL="https://github.com/infracloudio/cassandra.git"
+    echo "REPO_URL is empty. Assigning default value: $REPO_URL"
+fi
+
+if [ -n "$REPO_BRANCH" ]; then
+    # The variable is not empty, so it has a value
+    echo "REPO_BRANCH is not empty. Its value is: $REPO_BRANCH"
+else
+    # The variable is empty, so assign a default value
+    REPO_BRANCH="infracloud/cassandra-5.0"
+    echo "REPO_BRANCH is empty. Assigning default value: $REPO_BRANCH"
+fi
+
+if [ -n "$TEAR_DOWN" ]; then
+    # The variable is not empty, so it has a value
+    echo "TEAR_DOWN is not empty. Its value is: $TEAR_DOWN"
+else
+    # The variable is empty, so assign a default value
+    TEAR_DOWN=false
+    echo "TEAR_DOWN is empty. Assigning default value: $TEAR_DOWN"
+fi
+
+sed -e "/targets:/s|:.*$|: \"$TARGETS\"|" \
+    -e "/repositoryBranch:/s|:.*$|: \"$REPO_BRANCH\"|" \
+    -e "/repositoryUrl:/s|:.*$|: \"$REPO_URL\"|" ${CASSANDRA_DIR}/.jenkins/k8s/jenkins-deployment.yaml > ${CASSANDRA_DIR}/build/jenkins-deployment.yaml
+
+# get the job name from the DSL Job file
+JOB_NAME=$(grep -o "pipelineJob('\([^']*\)')" .jenkins/job/DslJob.jenkins | sed "s/pipelineJob('//;s/')//")
+
+
+echo "JOB_NAME = $JOB_NAME"
+
+# Add Helm Jenkins Operator repository
+echo "Adding Helm repository for Jenkins Operator..."
+helm repo add --namespace ${KUBE_NS} jenkins https://raw.githubusercontent.com/jenkinsci/kubernetes-operator/master/chart
+
+# Install Jenkins Operator using Helm
+echo "Installing Jenkins Operator..."
+helm upgrade --namespace ${KUBE_NS} --install jenkins-operator jenkins/jenkins-operator --set jenkins.enabled=false --set jenkins.backup.enabled=false --version 0.8.0-beta.2 
+
+while ! ( kubectl --namespace ${KUBE_NS} get pods | grep jenkins-operator | grep " 1/1 " | grep -q " Running" ) ; do
+        echo "Jenkins Operator installing. Waiting..."
+        sleep 5  # Adjust the polling interval as needed
+done
+
+echo "Jenkins Operator installed successfully!"
+
+kubectl apply --namespace ${KUBE_NS} -f ${CASSANDRA_DIR}/build/jenkins-deployment.yaml
+
+while ! ( kubectl --namespace ${KUBE_NS} get pods | grep jenkins-jenkins | grep " 1/1 " | grep -q " Running" ) ; do
+        echo "Jenkins installing. Waiting..."
+        sleep 5  # Adjust the polling interval as needed
+done
+
+kubectl rollout status deployment/jenkins-operator -n ${KUBE_NS}
+
+# Port-forward the Jenkins service to access it locally
+jenkins_pod=$(kubectl get pods -n ${KUBE_NS} -l jenkins-cr=jenkins -o jsonpath='{.items[0].metadata.name}')
+
+nohup kubectl port-forward svc/jenkins-operator-http-jenkins 8080:8080 > /dev/null >2&1 &
+sleep 5
+echo "port-forwarding running in background"
+# echo "To forward the Jenkins service to another terminal, open a new terminal window and run the following command:"
+# echo "kubectl port-forward -n ${KUBE_NS} $jenkins_pod 8080:8080"
+
+TOKEN=$(kubectl  get secret jenkins-operator-credentials-jenkins -o jsonpath="{.data.token}" | base64 --decode)
+
+# Trigger a new build and capture the response headers
+response_headers=$(curl -i -X POST http://localhost:8080/job/$JOB_NAME/buildWithParameters -u jenkins-operator:$TOKEN --data-urlencode "TEST_STAGES_TO_RUN=$INCLUDE_TEST_STAGE" 2>&1)
+
+echo "response_headers $response_headers"
+queue_url=$(echo "$response_headers" | grep -i "Location" | awk -F ": " '{print $2}' | tr -d '\r')
+queue_item_number=$(basename "$queue_url")
+
+# Construct the complete URL to retrieve build information
+queue_json_url="http://localhost:8080/queue/item/$queue_item_number/api/json"
+
+# Wait for the build number to become available (querying the API)
+build_number=""
+while [ -z "$build_number" ] || [ "$build_number" == "null" ]; do
+    build_info=$(curl -s "$queue_json_url" -u jenkins-operator:$TOKEN)
+    build_number=$(echo "$build_info" | jq -r '.executable.number')
+    if [ -z "$build_number" ] || [ "$build_number" == "null" ]; then
+        echo "Build number not available yet. Waiting..."
+        sleep 5  # Adjust the polling interval as needed
+    fi
+done
+
+echo "build_number $build_number"
+BUILD_DIR=/var/lib/jenkins/jobs/$JOB_NAME/builds
+POD_NAME=jenkins-jenkins
+LATEST_BUILD=$(kubectl exec -it $POD_NAME -- /bin/bash -c "ls -t $BUILD_DIR | head -n 1" | tr -d '\r')
+
+CONSOLE_LOG_FILE="$BUILD_DIR/$build_number/log"
+
+# Define a function to check if "FINISHED" is in the consoleLog
+check_finished() {
+    kubectl exec -n $KUBE_NS $POD_NAME -- cat "$CONSOLE_LOG_FILE" | grep -q "Finished"
+}
+
+# Continuously check for "FINISHED"
+while true; do
+    if check_finished; then
+        echo "Build has finished."
+        break
+    else
+        echo "Build is still in progress."
+    fi
+    sleep 5  # Adjust the sleep interval as needed
+done
+
+RESULTS_DIR="${CASSANDRA_DIR}/build/ci_${build_number}"
+mkdir -p ${RESULTS_DIR}
+kubectl cp -n $KUBE_NS $POD_NAME:${CONSOLE_LOG_FILE} ${RESULTS_DIR}/log
+
+if kubectl exec -n $KUBE_NS $POD_NAME -- test -e $BUILD_DIR/$build_number/junitResult.xml; then
+    kubectl cp -n $KUBE_NS $POD_NAME:$BUILD_DIR/$build_number/junitResult.xml ${RESULTS_DIR}/junitResult.xml
+else
+    # Display a message indicating that junitResult.xml is not generated
+    echo "junitResult.xml is not generated."
+fi
+
+
+# kill the port-forwarding process
+ps -elf | grep port-forward | head -n1 | awk -F " " '{ print $4 }' | xargs kill -9
+
+
+if [ $TEAR_DOWN ]; then
+    kubectl delete --namespace ${KUBE_NS} -f ${CASSANDRA_DIR}/build/jenkins-deployment.yaml
+    helm uninstall --namespace ${KUBE_NS} jenkins-operator
+fi