From 496574480eb56435d0cd722819f3a7f8819c7136 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 29 Aug 2024 12:04:12 -0400 Subject: [PATCH 1/6] chore: Update to use Protobuf 4.27.4 --- .kokoro/build.sh | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 8fafeb0e57..f677171d9f 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -27,6 +27,81 @@ source ${scriptDir}/common.sh java -version echo ${JOB_TYPE} +# Store the current Java version since the version may change when installing sdk-platform-java +current_java_home=$JAVA_HOME + +# testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java +# needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. +if [ ! -z "${JAVA11_HOME}" ]; then + export JAVA_HOME="${JAVA11_HOME}" + export PATH=${JAVA_HOME}/bin:$PATH +fi + +# Get the current proto runtime version used in this repo +CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | +sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { + //{ + s/\(.*\)<\/version>/\1/p + q + } +}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') +echo "The current proto version is: ${CURRENT_PROTO_VERSION}" + +# Find the latest proto runtime version available +LATEST_PROTO_VERSION="4.27.4" +echo "The latest proto version is: ${LATEST_PROTO_VERSION}" + +# Only reinstall shared-deps again to test for a newer proto version +if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then + pushd /tmp + git clone https://github.com/googleapis/sdk-platform-java.git + pushd sdk-platform-java + pushd gapic-generator-java-pom-parent + sed -i "/.*<\/protobuf.version>/s/\(.*\).*\(<\/protobuf.version>\)/\1${LATEST_PROTO_VERSION}\2/" pom.xml + # sdk-platform-java + popd + + pushd sdk-platform-java-config + # Get current Shared-Deps version in sdk-platform-java + SHARED_DEPS_VERSION=$(mvn -ntp help:effective-pom | + sed -n "/sdk-platform-java-config<\/artifactId>/,/<\/dependency>/ { + //{ + s/\(.*\)<\/version>/\1/p + q + } + }" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + echo "Shared-Deps Version: ${SHARED_DEPS_VERSION}" + # sdk-platform-java + popd + + mvn clean install -q -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -T 1C + # /tmp + popd + + # Back to the original directory of the repo + popd + # Find all the poms with a reference to shared-deps and update to the new local version + poms=($(find . -name pom.xml)) + for pom in "${poms[@]}"; do + if grep -q "sdk-platform-java-config" "${pom}"; then + echo "Updating the pom: ${pom} to use shared-deps version: ${SHARED_DEPS_VERSION}" + sed -i "/sdk-platform-java-config<\/artifactId>/,/<\/parent>/ s/.*<\/version>/$SHARED_DEPS_VERSION<\/version>/" "${pom}" +# xmlstarlet ed --inplace -N x="http://maven.apache.org/POM/4.0.0" \ +# -u "//x:project/x:parent[x:artifactId='sdk-platform-java-config']/x:version" \ +# -v "${SHARED_DEPS_VERSION}" \ +# "${pom}" + fi + done +fi + +# Reset back to the original Java version if changed +export JAVA_HOME="${current_java_home}" +export PATH=${JAVA_HOME}/bin:$PATH + # attempt to install 3 times with exponential backoff (starting with 10 seconds) retry_with_backoff 3 10 \ mvn install -B -V -U \ From 2e94b8b702dbb9fdbb233a8c62a114e231132b6b Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 16:44:53 -0400 Subject: [PATCH 2/6] chore: Test script --- .github/workflows/ci.yaml | 4 ++-- .kokoro/build.sh | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dbbecdf783..8fde9d2577 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,7 +52,7 @@ jobs: - uses: actions/setup-java@v2 with: distribution: temurin - java-version: 8 + java-version: 17 - run: java -version - run: .kokoro/build.bat env: @@ -101,7 +101,7 @@ jobs: - uses: actions/setup-java@v2 with: distribution: temurin - java-version: 8 + java-version: 17 - run: java -version - run: .kokoro/build.sh env: diff --git a/.kokoro/build.sh b/.kokoro/build.sh index f677171d9f..b1c39ac23c 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -30,13 +30,6 @@ echo ${JOB_TYPE} # Store the current Java version since the version may change when installing sdk-platform-java current_java_home=$JAVA_HOME -# testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java -# needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. -if [ ! -z "${JAVA11_HOME}" ]; then - export JAVA_HOME="${JAVA11_HOME}" - export PATH=${JAVA_HOME}/bin:$PATH -fi - # Get the current proto runtime version used in this repo CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { @@ -53,6 +46,13 @@ echo "The latest proto version is: ${LATEST_PROTO_VERSION}" # Only reinstall shared-deps again to test for a newer proto version if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then + # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java + # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. + if [ ! -z "${JAVA11_HOME}" ]; then + export JAVA_HOME="${JAVA11_HOME}" + export PATH=${JAVA_HOME}/bin:$PATH + fi + pushd /tmp git clone https://github.com/googleapis/sdk-platform-java.git pushd sdk-platform-java @@ -96,11 +96,11 @@ if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then # "${pom}" fi done -fi -# Reset back to the original Java version if changed -export JAVA_HOME="${current_java_home}" -export PATH=${JAVA_HOME}/bin:$PATH + # Reset back to the original Java version if changed + export JAVA_HOME="${current_java_home}" + export PATH=${JAVA_HOME}/bin:$PATH +fi # attempt to install 3 times with exponential backoff (starting with 10 seconds) retry_with_backoff 3 10 \ From 3ca6785f5a59963712ab3b08a4e3291265afd584 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 17:12:51 -0400 Subject: [PATCH 3/6] chore: Test script --- .../clirr-ignored-differences.xml | 67 +++++++++++++++++++ .../clirr-ignored-differences.xml | 67 +++++++++++++++++++ .../clirr-ignored-differences.xml | 67 +++++++++++++++++++ .../clirr-ignored-differences.xml | 67 +++++++++++++++++++ 4 files changed, 268 insertions(+) diff --git a/proto-google-cloud-bigquerystorage-v1/clirr-ignored-differences.xml b/proto-google-cloud-bigquerystorage-v1/clirr-ignored-differences.xml index b43cee05fb..d905949d80 100644 --- a/proto-google-cloud-bigquerystorage-v1/clirr-ignored-differences.xml +++ b/proto-google-cloud-bigquerystorage-v1/clirr-ignored-differences.xml @@ -16,4 +16,71 @@ com/google/cloud/bigquery/storage/v1/*OrBuilder boolean has*(*) + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * getDefaultInstanceForType() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * addRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * clearField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * clearOneof(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * clone() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * mergeUnknownFields(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * setField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * setRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1/** + * setUnknownFields(*) + ** + diff --git a/proto-google-cloud-bigquerystorage-v1alpha/clirr-ignored-differences.xml b/proto-google-cloud-bigquerystorage-v1alpha/clirr-ignored-differences.xml index 22bf9de1da..1870f8a936 100644 --- a/proto-google-cloud-bigquerystorage-v1alpha/clirr-ignored-differences.xml +++ b/proto-google-cloud-bigquerystorage-v1alpha/clirr-ignored-differences.xml @@ -16,4 +16,71 @@ com/google/cloud/bigquery/storage/v1alpha/*OrBuilder boolean has*(*) + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * getDefaultInstanceForType() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * addRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * clearField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * clearOneof(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * clone() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * mergeUnknownFields(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * setField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * setRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1alpha/** + * setUnknownFields(*) + ** + diff --git a/proto-google-cloud-bigquerystorage-v1beta1/clirr-ignored-differences.xml b/proto-google-cloud-bigquerystorage-v1beta1/clirr-ignored-differences.xml index 4fb3361c5e..f080138327 100644 --- a/proto-google-cloud-bigquerystorage-v1beta1/clirr-ignored-differences.xml +++ b/proto-google-cloud-bigquerystorage-v1beta1/clirr-ignored-differences.xml @@ -16,4 +16,71 @@ com/google/cloud/bigquery/storage/v1beta1/Storage$ReadRowsResponseOrBuilder * getSchemaCase() + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * getDefaultInstanceForType() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * addRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * clearField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * clearOneof(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * clone() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * mergeUnknownFields(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * setField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * setRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta1/** + * setUnknownFields(*) + ** + diff --git a/proto-google-cloud-bigquerystorage-v1beta2/clirr-ignored-differences.xml b/proto-google-cloud-bigquerystorage-v1beta2/clirr-ignored-differences.xml index f2cb475dfa..861d5dbaa0 100644 --- a/proto-google-cloud-bigquerystorage-v1beta2/clirr-ignored-differences.xml +++ b/proto-google-cloud-bigquerystorage-v1beta2/clirr-ignored-differences.xml @@ -16,4 +16,71 @@ com/google/cloud/bigquery/storage/v1beta2/*OrBuilder boolean has*(*) + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * getDefaultInstanceForType() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * addRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * clear() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * clearField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * clearOneof(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * clone() + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * mergeUnknownFields(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * setField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * setRepeatedField(*) + ** + + + 7006 + com/google/cloud/bigquery/storage/v1beta2/** + * setUnknownFields(*) + ** + From 6d1a40e812b8781b9c69ffd75e1c35889b4c812f Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 21:28:58 -0400 Subject: [PATCH 4/6] chore: Test script --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8fde9d2577..4f9e2f298e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,6 +53,7 @@ jobs: with: distribution: temurin java-version: 17 + - run: git config --system core.longpaths true - run: java -version - run: .kokoro/build.bat env: From 569963a84556b6775497ca424ca32272a9e12557 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 19 Sep 2024 11:20:36 -0400 Subject: [PATCH 5/6] chore: Update to Protobuf 4.28.2 --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index b1c39ac23c..98ddcc6b80 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -41,7 +41,7 @@ sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { echo "The current proto version is: ${CURRENT_PROTO_VERSION}" # Find the latest proto runtime version available -LATEST_PROTO_VERSION="4.27.4" +LATEST_PROTO_VERSION="4.28.2" echo "The latest proto version is: ${LATEST_PROTO_VERSION}" # Only reinstall shared-deps again to test for a newer proto version From fcbf170714ac0d6adccad9fbeb06b6f0be0e78e9 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 19 Dec 2024 14:38:50 -0500 Subject: [PATCH 6/6] ci: Use 4.27.4 --- .kokoro/build.sh | 76 +----------------------------------------------- 1 file changed, 1 insertion(+), 75 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 98ddcc6b80..3fb9504cdb 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -27,81 +27,6 @@ source ${scriptDir}/common.sh java -version echo ${JOB_TYPE} -# Store the current Java version since the version may change when installing sdk-platform-java -current_java_home=$JAVA_HOME - -# Get the current proto runtime version used in this repo -CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | -sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { - //{ - s/\(.*\)<\/version>/\1/p - q - } -}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') -echo "The current proto version is: ${CURRENT_PROTO_VERSION}" - -# Find the latest proto runtime version available -LATEST_PROTO_VERSION="4.28.2" -echo "The latest proto version is: ${LATEST_PROTO_VERSION}" - -# Only reinstall shared-deps again to test for a newer proto version -if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then - # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java - # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. - if [ ! -z "${JAVA11_HOME}" ]; then - export JAVA_HOME="${JAVA11_HOME}" - export PATH=${JAVA_HOME}/bin:$PATH - fi - - pushd /tmp - git clone https://github.com/googleapis/sdk-platform-java.git - pushd sdk-platform-java - pushd gapic-generator-java-pom-parent - sed -i "/.*<\/protobuf.version>/s/\(.*\).*\(<\/protobuf.version>\)/\1${LATEST_PROTO_VERSION}\2/" pom.xml - # sdk-platform-java - popd - - pushd sdk-platform-java-config - # Get current Shared-Deps version in sdk-platform-java - SHARED_DEPS_VERSION=$(mvn -ntp help:effective-pom | - sed -n "/sdk-platform-java-config<\/artifactId>/,/<\/dependency>/ { - //{ - s/\(.*\)<\/version>/\1/p - q - } - }" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - echo "Shared-Deps Version: ${SHARED_DEPS_VERSION}" - # sdk-platform-java - popd - - mvn clean install -q -ntp \ - -DskipTests=true \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -T 1C - # /tmp - popd - - # Back to the original directory of the repo - popd - # Find all the poms with a reference to shared-deps and update to the new local version - poms=($(find . -name pom.xml)) - for pom in "${poms[@]}"; do - if grep -q "sdk-platform-java-config" "${pom}"; then - echo "Updating the pom: ${pom} to use shared-deps version: ${SHARED_DEPS_VERSION}" - sed -i "/sdk-platform-java-config<\/artifactId>/,/<\/parent>/ s/.*<\/version>/$SHARED_DEPS_VERSION<\/version>/" "${pom}" -# xmlstarlet ed --inplace -N x="http://maven.apache.org/POM/4.0.0" \ -# -u "//x:project/x:parent[x:artifactId='sdk-platform-java-config']/x:version" \ -# -v "${SHARED_DEPS_VERSION}" \ -# "${pom}" - fi - done - - # Reset back to the original Java version if changed - export JAVA_HOME="${current_java_home}" - export PATH=${JAVA_HOME}/bin:$PATH -fi - # attempt to install 3 times with exponential backoff (starting with 10 seconds) retry_with_backoff 3 10 \ mvn install -B -V -U \ @@ -143,6 +68,7 @@ integration) -Dit.test=!ITBigQueryWrite*RetryTest \ -Dsurefire.failIfNoSpecifiedTests=false \ -Dfailsafe.failIfNoSpecifiedTests=false \ + -Dprotobuf.version="4.27.4" \ -fae \ verify RETURN_CODE=$?