diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f386b6f16..ef4a06a65 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -5,7 +5,7 @@
       key: "cuda-build-openmpi"
       agents:
         queue: "juliagpu"
-        cuda: "11.0"
+        cuda: "*"
       env:
         OPENMPI_VER: "4.1"
         OPENMPI_VER_FULL: "4.1.4"
@@ -44,61 +44,26 @@
 
     - wait
 
-    - label: "Tests -- Julia 1.6"
+    - label: "Tests -- Julia {{matrix.version}}"
+      matrix:
+        setup:
+          version:
+            - "1.6"
+            - "1.7"
+            - "1.8"
+            - "1.9"
+            - "1.10"
+      concurrency: 1
+      concurrency_group: mpi_cuda
       plugins:
         - JuliaCI/julia#v1:
-            version: "1.6"
+            version: "{{matrix.version}}"
             persist_depot_dirs: packages,artifacts,compiled
       agents:
         queue: "juliagpu"
-        cuda: "11.0"
+        cuda: "*"
       if: build.message !~ /\[skip tests\]/
-      timeout_in_minutes: 60
-      env:
-        JULIA_MPI_TEST_NPROCS: 2
-        JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
-        OMPI_ALLOW_RUN_AS_ROOT: 1
-        OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
-        OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
-        OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
-        JULIA_CUDA_MEMORY_POOL: "none"
-      commands: |
-        echo "--- Configure MPI"
-        buildkite-agent artifact download --step "cuda-build-openmpi" mpi-prefix.tar.gz .
-        mkdir -p $${JULIA_MPI_PATH}
-        tar -zxf mpi-prefix.tar.gz --strip-components 1 -C $${JULIA_MPI_PATH}
-        export PATH=$${JULIA_MPI_PATH}/bin:$${PATH}
-        export LD_LIBRARY_PATH=$${JULIA_MPI_PATH}/lib:$${LD_LIBRARY_PATH}
-
-        echo "--- Setup Julia packages"
-        julia --color=yes --project=. -e '
-            import Pkg
-            Pkg.develop(; path = joinpath(pwd(), "lib", "MPIPreferences"))
-            '
-        julia --color=yes --project=test -e '
-            using Pkg
-            Pkg.develop(path="lib/MPIPreferences")
-            using MPIPreferences
-            MPIPreferences.use_system_binary(export_prefs=true)
-            rm("test/Manifest.toml")
-            '
-
-        echo "+++ Run tests"
-        julia --color=yes --project=. -e '
-           import Pkg
-           Pkg.test("MPI"; test_args=["--backend=CUDA"])
-           '
-
-    - label: "Tests -- Julia latest"
-      plugins:
-        - JuliaCI/julia#v1:
-            version: "1"
-            persist_depot_dirs: packages,artifacts,compiled
-      agents:
-        queue: "juliagpu"
-        cuda: "11.0"
-      if: build.message !~ /\[skip tests\]/
-      timeout_in_minutes: 60
+      timeout_in_minutes: 90
       env:
         JULIA_MPI_TEST_NPROCS: 2
         JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
diff --git a/src/environment.jl b/src/environment.jl
index b13e673a7..52597c430 100644
--- a/src/environment.jl
+++ b/src/environment.jl
@@ -324,6 +324,9 @@ provides a mechanism to check, so it will return `false` with other implementati
 
 This can be overriden by setting the `JULIA_MPI_HAS_CUDA` environment variable to `true`
 or `false`.
+
+!!! note
+    For OpenMPI or OpenMPI-based implementations you first need to call [Init()](@ref).
 """
 function has_cuda()
     flag = get(ENV, "JULIA_MPI_HAS_CUDA", nothing)
diff --git a/test/runtests.jl b/test/runtests.jl
index 99b7d186c..2a2fdd7db 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -38,6 +38,11 @@ using DoubleFloats
     end
     CUDA.precompile_runtime()
     ArrayType = CUDA.CuArray
+
+    @info """
+    Running CUDA tests. Ensure that your MPI implementation is
+    CUDA-aware using `MPI.has_cuda` before reporting issues.
+    """
 elseif backend_name == "AMDGPU"
     Pkg.add("AMDGPU")
     ENV["JULIA_MPI_TEST_ARRAYTYPE"] = "ROCArray"