From 95c2db2a591a5354112a21b4dea545b79170168e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Palet?= Date: Wed, 27 Nov 2024 17:35:53 +0000 Subject: [PATCH 1/2] feat: Lint and test service SDK before pushing --- scripts/sdk-create-pr.sh | 42 ++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/scripts/sdk-create-pr.sh b/scripts/sdk-create-pr.sh index f55f7cf..5839145 100755 --- a/scripts/sdk-create-pr.sh +++ b/scripts/sdk-create-pr.sh @@ -11,19 +11,19 @@ BRANCH_PREFIX=$1 COMMIT_INFO=$2 if [ $# -lt 2 ]; then - echo "Not enough arguments supplied. Required: 'branch-prefix' 'commit-info'" + echo "! Not enough arguments supplied. Required: 'branch-prefix' 'commit-info'" exit 1 fi if type -p go >/dev/null; then : else - echo "Go not installed, unable to proceed." + echo "! Go not installed, unable to proceed." exit 1 fi if [ ! -d ${SDK_REPO_LOCAL_PATH} ]; then - echo "sdk to commit not found in root. Please run make generate-sdk" + echo "! SDK to commit not found in root. Please run make generate-sdk" exit 1 fi @@ -41,7 +41,8 @@ else fi # Create temp directory to work on -work_dir=$(mktemp -d) +work_dir="$ROOT_DIR/temp" +mkdir -p ${work_dir} if [[ ! ${work_dir} || -d {work_dir} ]]; then echo "Unable to create temporary directory" exit 1 @@ -51,7 +52,7 @@ fi trap "rm -rf ${work_dir}" EXIT mkdir ${work_dir}/git_repo # Where the git repo will be created -mkdir ${work_dir}/sdk_backup # Backup of the SDK to check for new modules +mkdir ${work_dir}/sdk_backup # Backup of the SDK to check for new modules mkdir ${work_dir}/sdk_to_push # Copy of SDK to push # Prepare SDK to push @@ -60,7 +61,7 @@ rm -rf ${work_dir}/sdk_to_push/.git # Initialize git repo cd ${work_dir}/git_repo -git clone ${REPO_URL_SSH} ./ +git clone ${REPO_URL_SSH} ./ --quiet git config user.name "${COMMIT_NAME}" git config user.email "${COMMIT_EMAIL}" @@ -74,37 +75,50 @@ for service_path in ${work_dir}/sdk_to_push/services/*; do # Removal of pulled data is necessary because the old version may have files # that were deleted in the new version rm -rf ./services/$service/* - cp -a ${work_dir}/sdk_to_push/services/$service/. ./services/$service + cp -a ${work_dir}/sdk_to_push/services/$service/. ./services/$service # Check for changes in the specific folder compared to main service_changes=$(git status --porcelain "services/$service") if [[ -n "$service_changes" ]]; then - echo "Committing changes for $service" + echo -e "\n>> Detected changes in $service service" + + # If lint or test fails for a service, we skip it and continue to the next one + make lint skip-non-generated-files=true service=$service || { + echo "! Linting failed for $service. THE UPDATE OF THIS SERVICE WILL BE SKIPPED." + continue + } + make test skip-non-generated-files=true service=$service || { + echo "! Testing failed for $service. THE UPDATE OF THIS SERVICE WILL BE SKIPPED." + continue + } + if [[ "$BRANCH_PREFIX" != "main" ]]; then git switch main # This is needed to create a new branch for the service without including the previously committed files branch="$BRANCH_PREFIX/$service" git switch -c "$branch" else branch=$BRANCH_PREFIX - fi - + fi + git add services/${service}/ if [ "${LANGUAGE}" == "go" ] && [ ! -d "${work_dir}/sdk_backup/services/${service}/" ]; then # Check if it is a newly added SDK module # go work use -r adds a use directive to the go.work file for dir, if it exists, and removes the use directory if the argument directory doesn’t exist # the -r flag examines subdirectories of dir recursively # this prevents errors if there is more than one new module in the SDK generation - go work use -r . + go work use -r . git add go.work fi - + if [[ "$branch" != "main" ]]; then + echo ">> Creating PR for $service" git commit -m "Generate $service" git push origin "$branch" gh pr create --title "Generator: Update SDK /services/$service" --body "$COMMIT_INFO" --head "$branch" --base "main" else + echo ">> Pushing changes for $service service..." git commit -m "Generate $service: $COMMIT_INFO" git push origin "$branch" - fi + fi fi -done \ No newline at end of file +done From aa64ff7ab8fc70aa3c48af151ad5e29ca0ee43ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Palet?= Date: Wed, 27 Nov 2024 17:36:38 +0000 Subject: [PATCH 2/2] feat: Beautify echo prints --- scripts/download-oas.sh | 8 ++++---- scripts/generate-sdk/generate-sdk.sh | 10 +++++----- scripts/generate-sdk/languages/go.sh | 7 +++---- scripts/generate-sdk/languages/python.sh | 18 +++++++++--------- scripts/project.sh | 4 ++-- 5 files changed, 23 insertions(+), 24 deletions(-) diff --git a/scripts/download-oas.sh b/scripts/download-oas.sh index e2c6f9e..ff59bce 100755 --- a/scripts/download-oas.sh +++ b/scripts/download-oas.sh @@ -20,7 +20,7 @@ fi # Create temp directory to clone OAS repo work_dir=$(mktemp -d) if [[ ! ${work_dir} || -d {work_dir} ]]; then - echo "Unable to create temporary directory" + echo "! Unable to create temporary directory" exit 1 fi trap "rm -rf ${work_dir}" EXIT # Delete temp directory on exit @@ -33,7 +33,7 @@ fi # Move oas to root level mkdir ${ROOT_DIR}/oas cd ${work_dir} -git clone ${OAS_REPO} +git clone ${OAS_REPO} --quiet for service_dir in ${work_dir}/${OAS_REPO_NAME}/services/*; do max_version_dir="" @@ -55,8 +55,8 @@ for service_dir in ${work_dir}/${OAS_REPO_NAME}/services/*; do if [[ ${version} == *alpha* ]]; then # To support initial integrations of the IaaS API in an Alpha state, we will temporarily use it to generate an IaaS Alpha SDK module # This check can be removed once the IaaS API moves all endpoints to Beta - if [[ ${service} == "iaas" ]]; then - mv -f ${dir}/*.json ${ROOT_DIR}/oas/iaasalpha.json + if [[ ${service} == "iaas" ]]; then + mv -f ${dir}/*.json ${ROOT_DIR}/oas/iaasalpha.json fi if [[ ${ALLOW_ALPHA} != "true" ]]; then continue diff --git a/scripts/generate-sdk/generate-sdk.sh b/scripts/generate-sdk/generate-sdk.sh index 88c7a45..e409048 100755 --- a/scripts/generate-sdk/generate-sdk.sh +++ b/scripts/generate-sdk/generate-sdk.sh @@ -65,30 +65,30 @@ jar_path="${GENERATOR_PATH}/openapi-generator-cli.jar" if [ -e ${jar_path} ] && [ $(java -jar ${jar_path} version) == ${GENERATOR_VERSION_NUMBER} ]; then : else - echo "Downloading OpenAPI generator (version ${GENERATOR_VERSION}) to ${GENERATOR_PATH}..." + echo "Downloading OpenAPI generator (version ${GENERATOR_VERSION})..." mkdir -p ${GENERATOR_PATH} - wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/${GENERATOR_VERSION_NUMBER}/openapi-generator-cli-${GENERATOR_VERSION_NUMBER}.jar -O ${jar_path} + wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/${GENERATOR_VERSION_NUMBER}/openapi-generator-cli-${GENERATOR_VERSION_NUMBER}.jar -O ${jar_path} --quiet echo "Download done." fi # Generate SDK for the specified language case "${LANGUAGE}" in go) - echo -e "\nGenerating the Go SDK...\n" + echo -e "\n>> Generating the Go SDK..." source ${LANGUAGE_GENERATORS_FOLDER_PATH}/${LANGUAGE}.sh # Usage: generate_go_sdk GENERATOR_PATH GIT_HOST GIT_USER_ID [GIT_REPO_ID] [SDK_REPO_URL] generate_go_sdk ${jar_path} ${GIT_HOST} ${GIT_USER_ID} ${GIT_REPO_ID} ${SDK_REPO_URL} ;; python) - echo -e "\nGenerating the Python SDK...\n" + echo -e "\n>> Generating the Python SDK..." source ${LANGUAGE_GENERATORS_FOLDER_PATH}/${LANGUAGE}.sh # Usage: generate_python_sdk GENERATOR_PATH GIT_HOST GIT_USER_ID [GIT_REPO_ID] [SDK_REPO_URL] generate_python_sdk ${jar_path} ${GIT_HOST} ${GIT_USER_ID} ${GIT_REPO_ID} ${SDK_REPO_URL} ;; *) - echo "SDK language not supported." + echo "! SDK language not supported." exit 1 ;; esac diff --git a/scripts/generate-sdk/languages/go.sh b/scripts/generate-sdk/languages/go.sh index cd8e08d..f60e7ad 100644 --- a/scripts/generate-sdk/languages/go.sh +++ b/scripts/generate-sdk/languages/go.sh @@ -62,7 +62,7 @@ generate_go_sdk() { if type -p go >/dev/null; then : else - echo "Go not installed, unable to proceed." + echo "! Go not installed, unable to proceed." exit 1 fi @@ -80,7 +80,7 @@ generate_go_sdk() { # Backup of the current state of the SDK services dir (services/) sdk_services_backup_dir=$(mktemp -d) if [[ ! ${sdk_services_backup_dir} || -d {sdk_services_backup_dir} ]]; then - echo "Unable to create temporary directory" + echo "! Unable to create temporary directory" exit 1 fi cleanup() { @@ -128,7 +128,7 @@ generate_go_sdk() { exit 1 fi - echo -e "\nGenerating \"${service}\" service..." + echo -e "\n>> Generating \"${service}\" service..." cd ${ROOT_DIR} GO_POST_PROCESS_FILE="gofmt -w" \ @@ -215,5 +215,4 @@ generate_go_sdk() { cd ${SDK_REPO_LOCAL_PATH} goimports -w ${SERVICES_FOLDER}/ make sync-tidy - } diff --git a/scripts/generate-sdk/languages/python.sh b/scripts/generate-sdk/languages/python.sh index 3164855..52d45c5 100644 --- a/scripts/generate-sdk/languages/python.sh +++ b/scripts/generate-sdk/languages/python.sh @@ -24,12 +24,12 @@ generate_python_sdk() { # Check required parameters if [[ -z ${GIT_HOST} ]]; then - echo "GIT_HOST not specified." + echo "! GIT_HOST not specified." exit 1 fi if [[ -z ${GIT_USER_ID} ]]; then - echo "GIT_USER_ID id not specified." + echo "! GIT_USER_ID id not specified." exit 1 fi @@ -46,7 +46,7 @@ generate_python_sdk() { # Prepare folders if [[ ! -d $SERVICES_FOLDER ]]; then - mkdir -p "$SERVICES_FOLDER" + mkdir -p "$SERVICES_FOLDER" fi # Clone SDK repo @@ -63,7 +63,7 @@ generate_python_sdk() { # Backup of the current state of the SDK services dir (services/) sdk_services_backup_dir=$(mktemp -d) if [[ ! ${sdk_services_backup_dir} || -d {sdk_services_backup_dir} ]]; then - echo "Unable to create temporary directory" + echo "! Unable to create temporary directory" exit 1 fi cleanup() { @@ -75,7 +75,7 @@ generate_python_sdk() { trap cleanup EXIT # Remove old contents of services dir (services/) - rm -rf ${SERVICES_FOLDER} + rm -rf ${SERVICES_FOLDER} # Generate SDK for each service for service_json in ${ROOT_DIR}/oas/*.json; do @@ -89,7 +89,7 @@ generate_python_sdk() { service=$(echo "${service}" | tr '[:upper:]' '[:lower:]') # convert upper case letters to lower case service=$(echo "${service}" | tr -d -c '[:alnum:]') # remove non-alphanumeric characters - echo "Generating \"${service}\" service..." + echo ">> Generating \"${service}\" service..." cd ${ROOT_DIR} mkdir -p "${SERVICES_FOLDER}/${service}/" @@ -113,11 +113,11 @@ generate_python_sdk() { rm -r "${SERVICES_FOLDER}/${service}/.openapi-generator/" rm "${SERVICES_FOLDER}/${service}/stackit/__init__.py" rm "${SERVICES_FOLDER}/${service}/.github/workflows/python.yml" - + # Create source layout mkdir "${SERVICES_FOLDER}/${service}/src" mv "${SERVICES_FOLDER}/${service}/stackit/" "${SERVICES_FOLDER}/${service}/src/" - + # If the service has a wait package files, move them inside the service folder if [ -d ${sdk_services_backup_dir}/${service}/src/wait ]; then echo "Found ${service} \"wait\" package" @@ -165,6 +165,6 @@ generate_python_sdk() { isort . autoimport --ignore-init-modules . black . - + done } diff --git a/scripts/project.sh b/scripts/project.sh index 8b3faa7..a29840a 100755 --- a/scripts/project.sh +++ b/scripts/project.sh @@ -27,8 +27,8 @@ elif [ "$action" = "tools" ]; then elif [ "${LANGUAGE}" == "python" ]; then pip install black==24.8.0 isort~=5.13.2 autoimport~=1.6.1 else - echo "Invalid language: `$LANGUAGE`, please use $0 help for help" + echo "! Invalid language: $($LANGUAGE), please use $0 help for help" fi else - echo "Invalid action: '$action', please use $0 help for help" + echo "! Invalid action: '$action', please use $0 help for help" fi