Lint and test service before pushing to main (#111)

* feat: Lint and test service SDK before pushing * feat: Beautify echo prints
stackitcloud · Nov 28, 2024 · 8810d4d · 8810d4d
1 parent fc09ddc
commit 8810d4d
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 38 deletions.
diff --git a/scripts/download-oas.sh b/scripts/download-oas.sh
@@ -20,7 +20,7 @@ fi
 # Create temp directory to clone OAS repo
 work_dir=$(mktemp -d)
 if [[ ! ${work_dir} || -d {work_dir} ]]; then
-    echo "Unable to create temporary directory"
+    echo "! Unable to create temporary directory"
     exit 1
 fi
 trap "rm -rf ${work_dir}" EXIT # Delete temp directory on exit
@@ -33,7 +33,7 @@ fi
 # Move oas to root level
 mkdir ${ROOT_DIR}/oas
 cd ${work_dir}
-git clone ${OAS_REPO}
+git clone ${OAS_REPO} --quiet
 
 for service_dir in ${work_dir}/${OAS_REPO_NAME}/services/*; do
     max_version_dir=""
@@ -55,8 +55,8 @@ for service_dir in ${work_dir}/${OAS_REPO_NAME}/services/*; do
             if [[ ${version} == *alpha* ]]; then
                 # To support initial integrations of the IaaS API in an Alpha state, we will temporarily use it to generate an IaaS Alpha SDK module
                 # This check can be removed once the IaaS API moves all endpoints to Beta
-                if [[ ${service} == "iaas" ]]; then 
-                    mv -f ${dir}/*.json ${ROOT_DIR}/oas/iaasalpha.json 
+                if [[ ${service} == "iaas" ]]; then
+                    mv -f ${dir}/*.json ${ROOT_DIR}/oas/iaasalpha.json
                 fi
                 if [[ ${ALLOW_ALPHA} != "true" ]]; then
                     continue

diff --git a/scripts/generate-sdk/generate-sdk.sh b/scripts/generate-sdk/generate-sdk.sh
@@ -65,30 +65,30 @@ jar_path="${GENERATOR_PATH}/openapi-generator-cli.jar"
 if [ -e ${jar_path} ] && [ $(java -jar ${jar_path} version) == ${GENERATOR_VERSION_NUMBER} ]; then
     :
 else
-    echo "Downloading OpenAPI generator (version ${GENERATOR_VERSION}) to ${GENERATOR_PATH}..."
+    echo "Downloading OpenAPI generator (version ${GENERATOR_VERSION})..."
     mkdir -p ${GENERATOR_PATH}
-    wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/${GENERATOR_VERSION_NUMBER}/openapi-generator-cli-${GENERATOR_VERSION_NUMBER}.jar -O ${jar_path}
+    wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/${GENERATOR_VERSION_NUMBER}/openapi-generator-cli-${GENERATOR_VERSION_NUMBER}.jar -O ${jar_path} --quiet
     echo "Download done."
 fi
 
 # Generate SDK for the specified language
 case "${LANGUAGE}" in
 go)
-    echo -e "\nGenerating the Go SDK...\n"
+    echo -e "\n>> Generating the Go SDK..."
 
     source ${LANGUAGE_GENERATORS_FOLDER_PATH}/${LANGUAGE}.sh
     # Usage: generate_go_sdk GENERATOR_PATH GIT_HOST GIT_USER_ID [GIT_REPO_ID] [SDK_REPO_URL]
     generate_go_sdk ${jar_path} ${GIT_HOST} ${GIT_USER_ID} ${GIT_REPO_ID} ${SDK_REPO_URL}
     ;;
 python)
-    echo -e "\nGenerating the Python SDK...\n"
+    echo -e "\n>> Generating the Python SDK..."
 
     source ${LANGUAGE_GENERATORS_FOLDER_PATH}/${LANGUAGE}.sh
     # Usage: generate_python_sdk GENERATOR_PATH GIT_HOST GIT_USER_ID [GIT_REPO_ID] [SDK_REPO_URL]
     generate_python_sdk ${jar_path} ${GIT_HOST} ${GIT_USER_ID} ${GIT_REPO_ID} ${SDK_REPO_URL}
     ;;
 *)
-    echo "SDK language not supported."
+    echo "! SDK language not supported."
     exit 1
     ;;
 esac
diff --git a/scripts/generate-sdk/languages/go.sh b/scripts/generate-sdk/languages/go.sh
@@ -62,7 +62,7 @@ generate_go_sdk() {
     if type -p go >/dev/null; then
         :
     else
-        echo "Go not installed, unable to proceed."
+        echo "! Go not installed, unable to proceed."
         exit 1
     fi
 
@@ -80,7 +80,7 @@ generate_go_sdk() {
     # Backup of the current state of the SDK services dir (services/)
     sdk_services_backup_dir=$(mktemp -d)
     if [[ ! ${sdk_services_backup_dir} || -d {sdk_services_backup_dir} ]]; then
-        echo "Unable to create temporary directory"
+        echo "! Unable to create temporary directory"
         exit 1
     fi
     cleanup() {
@@ -128,7 +128,7 @@ generate_go_sdk() {
             exit 1
         fi
 
-        echo -e "\nGenerating \"${service}\" service..."
+        echo -e "\n>> Generating \"${service}\" service..."
         cd ${ROOT_DIR}
 
         GO_POST_PROCESS_FILE="gofmt -w" \
@@ -215,5 +215,4 @@ generate_go_sdk() {
     cd ${SDK_REPO_LOCAL_PATH}
     goimports -w ${SERVICES_FOLDER}/
     make sync-tidy
-
 }
diff --git a/scripts/generate-sdk/languages/python.sh b/scripts/generate-sdk/languages/python.sh
@@ -24,12 +24,12 @@ generate_python_sdk() {
 
     # Check required parameters
     if [[ -z ${GIT_HOST} ]]; then
-        echo "GIT_HOST not specified."
+        echo "! GIT_HOST not specified."
         exit 1
     fi
 
     if [[ -z ${GIT_USER_ID} ]]; then
-        echo "GIT_USER_ID id not specified."
+        echo "! GIT_USER_ID id not specified."
         exit 1
     fi
 
@@ -46,7 +46,7 @@ generate_python_sdk() {
 
     # Prepare folders
     if [[ ! -d $SERVICES_FOLDER ]]; then
-      mkdir -p "$SERVICES_FOLDER"
+        mkdir -p "$SERVICES_FOLDER"
     fi
 
     # Clone SDK repo
@@ -63,7 +63,7 @@ generate_python_sdk() {
     # Backup of the current state of the SDK services dir (services/)
     sdk_services_backup_dir=$(mktemp -d)
     if [[ ! ${sdk_services_backup_dir} || -d {sdk_services_backup_dir} ]]; then
-        echo "Unable to create temporary directory"
+        echo "! Unable to create temporary directory"
         exit 1
     fi
     cleanup() {
@@ -75,7 +75,7 @@ generate_python_sdk() {
     trap cleanup EXIT
 
     # Remove old contents of services dir (services/)
-    rm -rf ${SERVICES_FOLDER}    
+    rm -rf ${SERVICES_FOLDER}
 
     # Generate SDK for each service
     for service_json in ${ROOT_DIR}/oas/*.json; do
@@ -89,7 +89,7 @@ generate_python_sdk() {
         service=$(echo "${service}" | tr '[:upper:]' '[:lower:]') # convert upper case letters to lower case
         service=$(echo "${service}" | tr -d -c '[:alnum:]')       # remove non-alphanumeric characters
 
-        echo "Generating \"${service}\" service..."
+        echo ">> Generating \"${service}\" service..."
         cd ${ROOT_DIR}
 
         mkdir -p "${SERVICES_FOLDER}/${service}/"
@@ -113,11 +113,11 @@ generate_python_sdk() {
         rm -r "${SERVICES_FOLDER}/${service}/.openapi-generator/"
         rm "${SERVICES_FOLDER}/${service}/stackit/__init__.py"
         rm "${SERVICES_FOLDER}/${service}/.github/workflows/python.yml"
-        
+
         # Create source layout
         mkdir "${SERVICES_FOLDER}/${service}/src"
         mv "${SERVICES_FOLDER}/${service}/stackit/" "${SERVICES_FOLDER}/${service}/src/"
-        
+
         # If the service has a wait package files, move them inside the service folder
         if [ -d ${sdk_services_backup_dir}/${service}/src/wait ]; then
             echo "Found ${service} \"wait\" package"
@@ -165,6 +165,6 @@ generate_python_sdk() {
         isort .
         autoimport --ignore-init-modules .
         black .
-        
+
     done
 }
diff --git a/scripts/project.sh b/scripts/project.sh
@@ -27,8 +27,8 @@ elif [ "$action" = "tools" ]; then
     elif [ "${LANGUAGE}" == "python" ]; then
         pip install black==24.8.0 isort~=5.13.2 autoimport~=1.6.1
     else
-        echo "Invalid language: `$LANGUAGE`, please use $0 help for help"
+        echo "! Invalid language: $($LANGUAGE), please use $0 help for help"
     fi
 else
-    echo "Invalid action: '$action', please use $0 help for help"
+    echo "! Invalid action: '$action', please use $0 help for help"
 fi
diff --git a/scripts/sdk-create-pr.sh b/scripts/sdk-create-pr.sh
@@ -11,19 +11,19 @@ BRANCH_PREFIX=$1
 COMMIT_INFO=$2
 
 if [ $# -lt 2 ]; then
-    echo "Not enough arguments supplied. Required: 'branch-prefix' 'commit-info'"
+    echo "! Not enough arguments supplied. Required: 'branch-prefix' 'commit-info'"
     exit 1
 fi
 
 if type -p go >/dev/null; then
     :
 else
-    echo "Go not installed, unable to proceed."
+    echo "! Go not installed, unable to proceed."
     exit 1
 fi
 
 if [ ! -d ${SDK_REPO_LOCAL_PATH} ]; then
-    echo "sdk to commit not found in root. Please run make generate-sdk"
+    echo "! SDK to commit not found in root. Please run make generate-sdk"
     exit 1
 fi
 
@@ -41,7 +41,8 @@ else
 fi
 
 # Create temp directory to work on
-work_dir=$(mktemp -d)
+work_dir="$ROOT_DIR/temp"
+mkdir -p ${work_dir}
 if [[ ! ${work_dir} || -d {work_dir} ]]; then
     echo "Unable to create temporary directory"
     exit 1
@@ -51,7 +52,7 @@ fi
 trap "rm -rf ${work_dir}" EXIT
 
 mkdir ${work_dir}/git_repo    # Where the git repo will be created
-mkdir ${work_dir}/sdk_backup # Backup of the SDK to check for new modules
+mkdir ${work_dir}/sdk_backup  # Backup of the SDK to check for new modules
 mkdir ${work_dir}/sdk_to_push # Copy of SDK to push
 
 # Prepare SDK to push
@@ -60,7 +61,7 @@ rm -rf ${work_dir}/sdk_to_push/.git
 
 # Initialize git repo
 cd ${work_dir}/git_repo
-git clone ${REPO_URL_SSH} ./
+git clone ${REPO_URL_SSH} ./ --quiet
 git config user.name "${COMMIT_NAME}"
 git config user.email "${COMMIT_EMAIL}"
 
@@ -74,37 +75,50 @@ for service_path in ${work_dir}/sdk_to_push/services/*; do
     # Removal of pulled data is necessary because the old version may have files
     # that were deleted in the new version
     rm -rf ./services/$service/*
-    cp -a ${work_dir}/sdk_to_push/services/$service/. ./services/$service 
+    cp -a ${work_dir}/sdk_to_push/services/$service/. ./services/$service
 
     # Check for changes in the specific folder compared to main
     service_changes=$(git status --porcelain "services/$service")
 
     if [[ -n "$service_changes" ]]; then
-        echo "Committing changes for $service" 
+        echo -e "\n>> Detected changes in $service service"
+
+        # If lint or test fails for a service, we skip it and continue to the next one
+        make lint skip-non-generated-files=true service=$service || {
+            echo "! Linting failed for $service. THE UPDATE OF THIS SERVICE WILL BE SKIPPED."
+            continue
+        }
+        make test skip-non-generated-files=true service=$service || {
+            echo "! Testing failed for $service. THE UPDATE OF THIS SERVICE WILL BE SKIPPED."
+            continue
+        }
+
         if [[ "$BRANCH_PREFIX" != "main" ]]; then
             git switch main # This is needed to create a new branch for the service without including the previously committed files
             branch="$BRANCH_PREFIX/$service"
             git switch -c "$branch"
         else
             branch=$BRANCH_PREFIX
-        fi            
-        
+        fi
+
         git add services/${service}/
         if [ "${LANGUAGE}" == "go" ] && [ ! -d "${work_dir}/sdk_backup/services/${service}/" ]; then # Check if it is a newly added SDK module
             # go work use -r adds a use directive to the go.work file for dir, if it exists, and removes the use directory if the argument directory doesn’t exist
             # the -r flag examines subdirectories of dir recursively
             # this prevents errors if there is more than one new module in the SDK generation
-            go work use -r . 
+            go work use -r .
             git add go.work
         fi
-        
+
         if [[ "$branch" != "main" ]]; then
+            echo ">> Creating PR for $service"
             git commit -m "Generate $service"
             git push origin "$branch"
             gh pr create --title "Generator: Update SDK /services/$service" --body "$COMMIT_INFO" --head "$branch" --base "main"
         else
+            echo ">> Pushing changes for $service service..."
             git commit -m "Generate $service: $COMMIT_INFO"
             git push origin "$branch"
-        fi   
+        fi
     fi
-done
+done