From 754a31eff6935f3199f0d5a3882c35904bd2a6ec Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 16 Oct 2024 10:31:18 +0300 Subject: [PATCH 1/5] ci: Fix the CI builds after Github Actions moving ubuntu-latest to 24.04 Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/deploy-docs.yml | 6 ++++++ .github/workflows/test-lang-csharp-ARM.yml | 8 ++++++-- .github/workflows/test-lang-csharp.yml | 8 ++++++-- .github/workflows/test-lang-java.yml | 4 ++-- .github/workflows/test-lang-js.yml | 4 ++-- share/docker/Dockerfile | 6 ++---- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 82dcc5d8b83..35be06e9d3d 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -203,6 +203,12 @@ jobs: with: fetch-depth: 0 + - name: Install dependencies + run: | + set -x + sudo apt-get update -q + sudo apt-get install -q -y subversion + - name: Download website uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/test-lang-csharp-ARM.yml b/.github/workflows/test-lang-csharp-ARM.yml index f7ad049fd17..4dcb496f887 100644 --- a/.github/workflows/test-lang-csharp-ARM.yml +++ b/.github/workflows/test-lang-csharp-ARM.yml @@ -63,7 +63,9 @@ jobs: run: ./build.sh lint - name: Test - run: ./build.sh test + run: | + export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 + ./build.sh test interop: runs-on: ubuntu-latest @@ -123,7 +125,9 @@ jobs: run: ./build.sh interop-data-generate - name: Run Interop Tests - run: ./build.sh interop-data-test + run: | + export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 + ./build.sh interop-data-test arm64: name: C# on Linux ARM64 diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml index 4efc70f3ff8..b7d0a76c938 100644 --- a/.github/workflows/test-lang-csharp.yml +++ b/.github/workflows/test-lang-csharp.yml @@ -63,7 +63,9 @@ jobs: run: ./build.sh lint - name: Test - run: ./build.sh test + run: | + export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 + ./build.sh test interop: runs-on: ubuntu-latest @@ -123,4 +125,6 @@ jobs: run: ./build.sh interop-data-generate - name: Run Interop Tests - run: ./build.sh interop-data-test + run: | + export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 + ./build.sh interop-data-test diff --git a/.github/workflows/test-lang-java.yml b/.github/workflows/test-lang-java.yml index bb427605767..fd0ed28f174 100644 --- a/.github/workflows/test-lang-java.yml +++ b/.github/workflows/test-lang-java.yml @@ -122,8 +122,8 @@ jobs: - name: 'Install Python Dependencies' run: | - python3 -m pip install --upgrade pip setuptools tox - python3 -m pip install python-snappy zstandard + python3 -m pip install --break-system-packages --upgrade pip setuptools tox + python3 -m pip install --break-system-packages python-snappy zstandard - name: 'Setup C# for Generating Interop Data' uses: actions/setup-dotnet@v4 diff --git a/.github/workflows/test-lang-js.yml b/.github/workflows/test-lang-js.yml index 20f2cc09f4c..60b06ddeb3c 100644 --- a/.github/workflows/test-lang-js.yml +++ b/.github/workflows/test-lang-js.yml @@ -96,8 +96,8 @@ jobs: libzstd-dev - name: Install Python Dependencies run: | - python3 -m pip install --upgrade pip setuptools tox - python3 -m pip install python-snappy zstandard + python3 -m pip install --break-system-packages --upgrade pip setuptools tox + python3 -m pip install --break-system-packages python-snappy zstandard - name: Create Interop Data Directory working-directory: . diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile index 66972b16822..6c1d7127b5b 100644 --- a/share/docker/Dockerfile +++ b/share/docker/Dockerfile @@ -191,7 +191,7 @@ RUN curl -sSL https://cpanmin.us \ && rm -rf .cpanm # Install Python3 -ENV PATH="${PATH}:/opt/pypy3.9/bin:/opt/pypy3.10/bin" \ +ENV PATH="${PATH}:/opt/pypy3.10/bin" \ PIP_NO_CACHE_DIR=off # https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope @@ -201,11 +201,9 @@ RUN case "${BUILDARCH:?}" in \ *) pypyarch=linux64;; \ esac \ && cd /opt \ - && for url in https://downloads.python.org/pypy/pypy3.9-v7.3.12-"$pypyarch".tar.bz2 \ - https://downloads.python.org/pypy/pypy3.10-v7.3.12-"$pypyarch".tar.bz2; \ + && for url in https://downloads.python.org/pypy/pypy3.10-v7.3.12-"$pypyarch".tar.bz2; \ do curl -fsSL "$url" | tar -xvjpf -; \ done \ - && ln -s pypy3.9* pypy3.9 \ && ln -s pypy3.10* pypy3.10 RUN python3 -m pip install --upgrade pip setuptools wheel \ From fc7f844a2d4251e9d298d7ac02725ee930fe1d8b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 16 Oct 2024 10:38:09 +0300 Subject: [PATCH 2/5] Try with libicu-dev instead of DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-lang-csharp-ARM.yml | 12 ++++-------- .github/workflows/test-lang-csharp.yml | 12 ++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-lang-csharp-ARM.yml b/.github/workflows/test-lang-csharp-ARM.yml index 4dcb496f887..554ee1a1e91 100644 --- a/.github/workflows/test-lang-csharp-ARM.yml +++ b/.github/workflows/test-lang-csharp-ARM.yml @@ -40,7 +40,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev + run: sudo apt-get install -y libzstd-dev libicu-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 @@ -63,9 +63,7 @@ jobs: run: ./build.sh lint - name: Test - run: | - export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 - ./build.sh test + run: ./build.sh test interop: runs-on: ubuntu-latest @@ -74,7 +72,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev + run: sudo apt-get install -y libzstd-dev libicu-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 @@ -125,9 +123,7 @@ jobs: run: ./build.sh interop-data-generate - name: Run Interop Tests - run: | - export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 - ./build.sh interop-data-test + run: ./build.sh interop-data-test arm64: name: C# on Linux ARM64 diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml index b7d0a76c938..d5bc3bcb077 100644 --- a/.github/workflows/test-lang-csharp.yml +++ b/.github/workflows/test-lang-csharp.yml @@ -40,7 +40,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev + run: sudo apt-get install -y libzstd-dev libicu-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 @@ -63,9 +63,7 @@ jobs: run: ./build.sh lint - name: Test - run: | - export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 - ./build.sh test + run: ./build.sh test interop: runs-on: ubuntu-latest @@ -74,7 +72,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev + run: sudo apt-get install -y libzstd-dev libicu-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 @@ -125,6 +123,4 @@ jobs: run: ./build.sh interop-data-generate - name: Run Interop Tests - run: | - export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 - ./build.sh interop-data-test + run: ./build.sh interop-data-test From f1346e2ff3810fe9f0b548b25c5a148b87c805cd Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 16 Oct 2024 11:18:19 +0300 Subject: [PATCH 3/5] Revert the changes for CSharp. They could be taken care in a separate PR Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-lang-csharp-ARM.yml | 6 +++--- .github/workflows/test-lang-csharp.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-lang-csharp-ARM.yml b/.github/workflows/test-lang-csharp-ARM.yml index 554ee1a1e91..0d1afd7f200 100644 --- a/.github/workflows/test-lang-csharp-ARM.yml +++ b/.github/workflows/test-lang-csharp-ARM.yml @@ -40,7 +40,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev libicu-dev + run: sudo apt-get install -y libzstd-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 @@ -72,7 +72,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev libicu-dev + run: sudo apt-get install -y libzstd-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 @@ -144,7 +144,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update -q - sudo apt-get install -q -y wget libzstd-dev libicu-dev + sudo apt-get install -q -y wget libzstd-dev wget https://dot.net/v1/dotnet-install.sh bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1 bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0 diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml index d5bc3bcb077..4efc70f3ff8 100644 --- a/.github/workflows/test-lang-csharp.yml +++ b/.github/workflows/test-lang-csharp.yml @@ -40,7 +40,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev libicu-dev + run: sudo apt-get install -y libzstd-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 @@ -72,7 +72,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd-dev libicu-dev + run: sudo apt-get install -y libzstd-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v4 From ac8b33b55fb4380f80ce2ea4b440f469f106727e Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 16 Oct 2024 11:55:35 +0300 Subject: [PATCH 4/5] Run C# tests on ubuntu-22.04 to fix the CI issues Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/test-lang-csharp-ARM.yml | 4 ++-- .github/workflows/test-lang-csharp.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-lang-csharp-ARM.yml b/.github/workflows/test-lang-csharp-ARM.yml index 0d1afd7f200..501a56b9c9b 100644 --- a/.github/workflows/test-lang-csharp-ARM.yml +++ b/.github/workflows/test-lang-csharp-ARM.yml @@ -34,7 +34,7 @@ concurrency: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -66,7 +66,7 @@ jobs: run: ./build.sh test interop: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml index 4efc70f3ff8..3e13e68772f 100644 --- a/.github/workflows/test-lang-csharp.yml +++ b/.github/workflows/test-lang-csharp.yml @@ -34,7 +34,7 @@ concurrency: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -66,7 +66,7 @@ jobs: run: ./build.sh test interop: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From b981fba85420f8383b3fe448c9ec2f9128fa1f5d Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 16 Oct 2024 13:41:44 +0300 Subject: [PATCH 5/5] Do not try to run tests for langs without build.sh (such as lang/rust/) Signed-off-by: Martin Tzvetanov Grigorov --- share/docker/run-tests.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/share/docker/run-tests.sh b/share/docker/run-tests.sh index a585a413600..74696653d9e 100755 --- a/share/docker/run-tests.sh +++ b/share/docker/run-tests.sh @@ -25,7 +25,9 @@ set -e for lang in /avro/lang/*/ do - headline "Run tests: $lang" - cd "$lang" - ./build.sh lint test + if [ -f "${lang}/build.sh" ]; then + headline "Run tests: $lang" + cd "$lang" + ./build.sh lint test + fi done