diff --git a/.github/workflows/avro-compatibility.yml b/.github/workflows/avro-compatibility.yml
deleted file mode 100644
index e15f892d9c8..00000000000
--- a/.github/workflows/avro-compatibility.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: Avro Compatibility Tests
-on: [push, pull_request]
-jobs:
-  test:
-    name: "Compatibility: avro ${{ matrix.avro }}/gradle ${{ matrix.gradle }}"
-    runs-on: "ubuntu-latest"
-    strategy:
-      matrix:
-        avro: ["1.11.0", "1.11.1"]
-        gradle: ["5.1", "7.6"]
-        java: ["8"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}
diff --git a/.github/workflows/gradle-compatibility.yml b/.github/workflows/gradle-compatibility.yml
deleted file mode 100644
index 0cea04c86ad..00000000000
--- a/.github/workflows/gradle-compatibility.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-name: Gradle Compatibility Tests
-on: [push, pull_request]
-jobs:
-  test:
-    name: "Compatibility: gradle ${{ matrix.gradle }}/java ${{ matrix.java }}"
-    runs-on: "ubuntu-latest"
-    strategy:
-      matrix:
-        avro: ["1.11.0"]
-        gradle: [
-          "5.1", "5.1.1", "5.2", "5.2.1", "5.3", "5.3.1", "5.4", "5.4.1", "5.5", "5.5.1", "5.6", "5.6.1", "5.6.2", "5.6.3", "5.6.4",
-          "6.0", "6.0.1", "6.1", "6.1.1", "6.2", "6.2.1", "6.2.2", "6.3", "6.4", "6.4.1", "6.5", "6.5.1", "6.6", "6.6.1", "6.7", "6.7.1",
-          "6.8", "6.8.1", "6.8.2", "6.8.3", "6.9", "6.9.1", "6.9.2", "6.9.3",
-          "7.0", "7.0.1", "7.0.2", "7.1", "7.1.1", "7.2", "7.3", "7.3.1", "7.3.2", "7.3.3", "7.4", "7.4.1", "7.4.2", "7.5", "7.5.1", "7.6"
-          # See here for latest versions: https://services.gradle.org/versions/
-        ]
-        java: ["8", "11"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}
diff --git a/.github/workflows/java-compatibility.yml b/.github/workflows/java-compatibility.yml
deleted file mode 100644
index dcaf0356b61..00000000000
--- a/.github/workflows/java-compatibility.yml
+++ /dev/null
@@ -1,102 +0,0 @@
-# See https://docs.gradle.org/current/userguide/compatibility.html
-name: Java Compatibility Tests
-on: [push, pull_request]
-jobs:
-  java8-12:
-    name: "Compatibility: java ${{ matrix.java }}/gradle ${{ matrix.gradle }}"
-    runs-on: "ubuntu-latest"
-    strategy:
-      matrix:
-        avro: ["1.11.0"]
-        gradle: ["5.1", "7.6"]
-        java: ["8", "11"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}
-  java17:
-    name: "Compatibility: java ${{ matrix.java }}/gradle ${{ matrix.gradle }}"
-    runs-on: "ubuntu-latest"
-    strategy:
-      matrix:
-        avro: ["1.11.0"]
-        gradle: ["7.3", "7.6"] # See here for latest versions: https://services.gradle.org/versions/
-        java: ["17"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}
-  java18:
-    name: "Compatibility: java ${{ matrix.java }}/gradle ${{ matrix.gradle }}"
-    runs-on: "ubuntu-latest"
-    strategy:
-      matrix:
-        avro: ["1.11.0"]
-        gradle: ["7.5", "7.6"] # See here for latest versions: https://services.gradle.org/versions/
-        java: ["18"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}
-  java-19:
-    name: "Compatibility: java ${{ matrix.java }}/gradle ${{ matrix.gradle }}"
-    runs-on: "ubuntu-latest"
-    strategy:
-      matrix:
-        avro: ["1.11.0"]
-        gradle: ["7.6"] # See here for latest versions: https://services.gradle.org/versions/
-        java: ["19"]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        continue-on-error: true
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}
-  java-ea:
-    name: "Compatibility: java ${{ matrix.java }}/gradle ${{ matrix.gradle }}"
-    runs-on: "ubuntu-latest"
-    strategy:
-      matrix:
-        avro: ["1.11.0"]
-        gradle: ["7.6"] # See here for latest versions: https://services.gradle.org/versions/
-        java: ["20-ea"]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        continue-on-error: true
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}
diff --git a/.github/workflows/os-compatibility.yml b/.github/workflows/os-compatibility.yml
deleted file mode 100644
index c585d247e0d..00000000000
--- a/.github/workflows/os-compatibility.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-name: OS Compatibility
-on: [push, pull_request]
-jobs:
-  build:
-    name: "Compatibility: ${{ matrix.os }}"
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        java: [11]
-        os: [ubuntu-latest, windows-latest, macOS-latest] # All supported OS
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: zulu
-          java-version: ${{ matrix.java }}
-      - uses: gradle/gradle-build-action@v2
-        with:
-          build-root-directory: lang/java/gradle-plugin
-          gradle-executable: lang/java/gradle-plugin/gradlew
-          arguments: test
diff --git a/.github/workflows/test-lang-java-gradle-plugin.yml b/.github/workflows/test-lang-java-gradle-plugin.yml
new file mode 100644
index 00000000000..bf7da27e213
--- /dev/null
+++ b/.github/workflows/test-lang-java-gradle-plugin.yml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: 'Gradle Plugin Compatibility Tests'
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+    paths:
+      - .github/workflows/test-gradle-plugin-compatibility.yml
+      - lang/java/gradle-plugin/**
+
+defaults:
+  run:
+    working-directory: lang/java/gradle-plugin
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  os-compatibility:
+    name: 'OS Compatibility: ${{ matrix.os }}'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
+        with:
+          distribution: temurin
+          java-version: 11
+      - uses: gradle/gradle-build-action@v2
+        with:
+          build-root-directory: lang/java/gradle-plugin
+          gradle-executable: lang/java/gradle-plugin/gradlew
+          arguments: test
+
+  version-compatibility:
+    name: 'Version Compatibility: Java ${{ matrix.java }}, Gradle ${{ matrix.gradle }}, Avro ${{ matrix.avro }}'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [11, 17, 21]
+        gradle: [5.6.4, 6.9.3, 7.6]
+        avro: [1.11.3]
+        exclude:
+          - java: 17
+            gradle: 5.6.4
+          - java: 19
+            gradle: 5.6.4
+          - java: 19
+            gradle: 6.9.3
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java }}
+      - uses: gradle/gradle-build-action@v2
+        with:
+          build-root-directory: lang/java/gradle-plugin
+          gradle-executable: lang/java/gradle-plugin/gradlew
+          arguments: testCompatibility -PavroVersion=${{ matrix.avro }} -PgradleVersion=${{ matrix.gradle }}