diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 80c22deb8..e3835d727 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,163 +1,137 @@
-steps:
-  - label: "Build OpenMPI -- CUDA"
-    agents:
-      queue: "juliagpu"
-      cuda: "11.0"
-    env:
-      OPENMPI_VER: "4.0"
-      OPENMPI_VER_FULL: "4.0.3"
-      UCX_VER: "1.12.1"
-      CCACHE_DIR: "/root/ccache"
-    commands: |
-      echo "--- Install packages"
-      apt-get install --yes --no-install-recommends curl ccache
-      export PATH="/usr/lib/ccache/:$$PATH"
+ - group: "CUDA"
+   key: "cuda"
+   steps:
+    - label: "Build OpenMPI"
+      key: "cuda-build-openmpi"
+      agents:
+        queue: "juliagpu"
+        cuda: "11.0"
+      env:
+        OPENMPI_VER: "4.0"
+        OPENMPI_VER_FULL: "4.0.3"
+        UCX_VER: "1.12.1"
+        CCACHE_DIR: "/root/ccache"
+      commands: |
+        echo "--- Install packages"
+        apt-get install --yes --no-install-recommends curl ccache
+        export PATH="/usr/lib/ccache/:$$PATH"
 
-      echo "--- Build UCX"
-      curl -L https://github.com/openucx/ucx/releases/download/v$${UCX_VER}/ucx-$${UCX_VER}.tar.gz --output ucx.tar.gz
-      tar -zxf ucx.tar.gz
-      pushd ucx-*
-      ./configure --with-cuda=/usr/local/cuda --enable-mt --prefix=$$(realpath ../mpi-prefix)
-      make -j
-      make install
-      popd
+        echo "--- Build UCX"
+        curl -L https://github.com/openucx/ucx/releases/download/v$${UCX_VER}/ucx-$${UCX_VER}.tar.gz --output ucx.tar.gz
+        tar -zxf ucx.tar.gz
+        pushd ucx-*
+        ./configure --with-cuda=/usr/local/cuda --enable-mt --prefix=$$(realpath ../mpi-prefix)
+        make -j
+        make install
+        popd
 
-      echo "--- Build OpenMPI"
-      curl -L https://download.open-mpi.org/release/open-mpi/v$${OPENMPI_VER}/openmpi-$${OPENMPI_VER_FULL}.tar.gz --output openmpi.tar.gz
-      tar -zxf openmpi.tar.gz
-      pushd openmpi-$${OPENMPI_VER_FULL}
-      ./configure --with-ucx=$$(realpath ../mpi-prefix) --prefix=$$(realpath ../mpi-prefix)
-      make -j
-      make install
-      popd
+        echo "--- Build OpenMPI"
+        curl -L https://download.open-mpi.org/release/open-mpi/v$${OPENMPI_VER}/openmpi-$${OPENMPI_VER_FULL}.tar.gz --output openmpi.tar.gz
+        tar -zxf openmpi.tar.gz
+        pushd openmpi-$${OPENMPI_VER_FULL}
+        ./configure --with-ucx=$$(realpath ../mpi-prefix) --with-cuda=/usr/local/cuda --prefix=$$(realpath ../mpi-prefix)
+        make -j
+        make install
+        popd
 
-      echo "--- Package prefix"
-      tar -zcf mpi-prefix.tar.gz mpi-prefix/
+        echo "--- Package prefix"
+        tar -zcf mpi-prefix.tar.gz mpi-prefix/
 
-      echo "--- ccache stats"
-      ccache -s
-    artifact_paths:
-      - "mpi-prefix.tar.gz"
-  - label: "Build OpenMPI -- ROCM"
-    agents:
-      queue: "juliagpu"
-      rocm: "*" # todo fix ROCM version
-    env:
-      OPENMPI_VER: "4.0"
-      OPENMPI_VER_FULL: "4.0.3"
-      UCX_VER: "1.12.1"
-      CCACHE_DIR: "/root/ccache"
-    commands: |
-      echo "--- Install packages"
-      apt-get install --yes --no-install-recommends curl ccache
-      export PATH="/usr/lib/ccache/:$$PATH"
+        echo "--- ccache stats"
+        ccache -s
+      artifact_paths:
+        - "mpi-prefix.tar.gz"
 
-      echo "--- Build UCX"
-      curl -L https://github.com/openucx/ucx/releases/download/v$${UCX_VER}/ucx-$${UCX_VER}.tar.gz --output ucx.tar.gz
-      tar -zxf ucx.tar.gz
-      pushd ucx-*
-      ./configure --with-rocm --enable-mt --prefix=$$(realpath ../mpi-prefix)
-      make -j
-      make install
-      popd
+    - wait
+    
+    - label: "Tests -- Julia 1.6"
+      plugins:
+        - JuliaCI/julia#v1:
+            version: "1.6"
+            persist_depot_dirs: packages,artifacts,compiled
+      agents:
+        queue: "juliagpu"
+        cuda: "11.0"
+      if: build.message !~ /\[skip tests\]/
+      timeout_in_minutes: 60
+      env:
+        JULIA_MPI_TEST_ARRAYTYPE: CuArray
+        JULIA_MPI_TEST_NPROCS: 2
+        JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
+        OMPI_ALLOW_RUN_AS_ROOT: 1
+        OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+        OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
+        OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
+        JULIA_CUDA_MEMORY_POOL: "none"
+      commands: |
+        echo "--- Configure MPI"
+        buildkite-agent artifact download --step "cuda-build-openmpi" mpi-prefix.tar.gz .
+        mkdir -p $${JULIA_MPI_PATH}
+        tar -zxf mpi-prefix.tar.gz --strip-components 1 -C $${JULIA_MPI_PATH}
+        export PATH=$${JULIA_MPI_PATH}/bin:$${PATH}
+        export LD_LIBRARY_PATH=$${JULIA_MPI_PATH}/lib:$${LD_LIBRARY_PATH}
 
-      echo "--- Build OpenMPI"
-      curl -L https://download.open-mpi.org/release/open-mpi/v$${OPENMPI_VER}/openmpi-$${OPENMPI_VER_FULL}.tar.gz --output openmpi.tar.gz
-      tar -zxf openmpi.tar.gz
-      pushd openmpi-*
-      ./configure --with-ucx=$$(realpath ../mpi-prefix) --prefix=$$(realpath ../mpi-prefix)
-      make -j
-      make install
-      popd
+        echo "--- Setup Julia packages"
+        julia --color=yes --project=. -e '
+            import Pkg
+            Pkg.develop(; path = joinpath(pwd(), "lib", "MPIPreferences"))
+            '
+        julia --color=yes --project=test -e '
+            using Pkg
+            Pkg.develop(path="lib/MPIPreferences")
+            using MPIPreferences
+            MPIPreferences.use_system_binary(export_prefs=true)
+            rm("test/Manifest.toml")
+            '
 
-      echo "--- Package prefix"
-      tar -zcf mpi-prefix.tar.gz mpi-prefix/
+        echo "+++ Run tests"
+        julia --color=yes --project=. -e '
+           import Pkg
+           Pkg.test("MPI")
+           '
 
-      echo "--- ccache stats"
-      ccache -s
-    artifact_paths:
-      - "mpi-prefix.tar.gz"
-  - wait: ~
-  - label: "CUDA -- 1.6"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1.6"
-          persist_depot_dirs: packages,artifacts,compiled
-    agents:
-      queue: "juliagpu"
-      cuda: "11.0"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 60
-    env:
-      JULIA_MPI_TEST_ARRAYTYPE: CuArray
-      JULIA_MPI_TEST_NPROCS: 2
-      JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
-      OMPI_ALLOW_RUN_AS_ROOT: 1
-      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
-      OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
-      OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
-      JULIA_CUDA_MEMORY_POOL: "none"
-    soft_fail: true
-    commands: |
-      echo "--- Configure MPI"
-      buildkite-agent artifact download --step "Build OpenMPI -- CUDA" mpi-prefix.tar.gz .
-      mkdir -p $${JULIA_MPI_PATH}
-      tar -zxf mpi-prefix.tar.gz --strip-components 1 -C $${JULIA_MPI_PATH}
-      export PATH=$${JULIA_MPI_PATH}/bin:$${PATH}
-      export LD_LIBRARY_PATH=$${JULIA_MPI_PATH}/lib:$${LD_LIBRARY_PATH}
+    - label: "Tests -- Julia 1.7"
+      plugins:
+        - JuliaCI/julia#v1:
+            version: "1.7"
+            persist_depot_dirs: packages,artifacts,compiled
+      agents:
+        queue: "juliagpu"
+        cuda: "11.0"
+      if: build.message !~ /\[skip tests\]/
+      timeout_in_minutes: 60
+      env:
+        JULIA_MPI_TEST_ARRAYTYPE: CuArray
+        JULIA_MPI_TEST_NPROCS: 2
+        JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
+        OMPI_ALLOW_RUN_AS_ROOT: 1
+        OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+        OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
+        OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
+        JULIA_CUDA_MEMORY_POOL: "none"
+      commands: |
+        echo "--- Configure MPI"
+        buildkite-agent artifact download --step "cuda-build-openmpi" mpi-prefix.tar.gz .
+        mkdir -p $${JULIA_MPI_PATH}
+        tar -zxf mpi-prefix.tar.gz --strip-components 1 -C $${JULIA_MPI_PATH}
+        export PATH=$${JULIA_MPI_PATH}/bin:$${PATH}
+        export LD_LIBRARY_PATH=$${JULIA_MPI_PATH}/lib:$${LD_LIBRARY_PATH}
 
-      echo "--- Setup Julia packages"
-      julia --color=yes --project=test -e '
-          using Pkg
-          Pkg.develop(path="lib/MPIPreferences")
-          using MPIPreferences
-          MPIPreferences.use_system_binary(export_prefs=true)
-          rm("test/Manifest.toml")'
-      julia -e 'import Pkg; Pkg.develop(; path = joinpath(pwd(), "lib", "MPIPreferences"))'
-      julia -e 'import Pkg; Pkg.develop(; path = pwd())'
-      julia -e 'import Pkg; Pkg.precompile()'
+        echo "--- Setup Julia packages"
+        julia --color=yes --project=. -e '
+            import Pkg
+            Pkg.develop(; path = joinpath(pwd(), "lib", "MPIPreferences"))
+            '
+        julia --color=yes --project=test -e '
+            using Pkg
+            Pkg.develop(path="lib/MPIPreferences")
+            using MPIPreferences
+            MPIPreferences.use_system_binary(export_prefs=true)
+            rm("test/Manifest.toml")
+            '
 
-      echo "+++ Run tests"
-      julia -e 'import Pkg; Pkg.test("MPI")'
-
-  - label: "CUDA -- 1.7"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1.7"
-          persist_depot_dirs: packages,artifacts,compiled
-    agents:
-      queue: "juliagpu"
-      cuda: "11.0"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 60
-    env:
-      JULIA_MPI_TEST_ARRAYTYPE: CuArray
-      JULIA_MPI_TEST_NPROCS: 2
-      JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
-      OMPI_ALLOW_RUN_AS_ROOT: 1
-      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
-      OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
-      OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
-      JULIA_CUDA_MEMORY_POOL: "none"
-    soft_fail: true
-    commands: |
-      echo "--- Configure MPI"
-      buildkite-agent artifact download --step "Build OpenMPI -- CUDA" mpi-prefix.tar.gz .
-      mkdir -p $${JULIA_MPI_PATH}
-      tar -zxf mpi-prefix.tar.gz --strip-components 1 -C $${JULIA_MPI_PATH}
-      export PATH=$${JULIA_MPI_PATH}/bin:$${PATH}
-      export LD_LIBRARY_PATH=$${JULIA_MPI_PATH}/lib:$${LD_LIBRARY_PATH}
-
-      echo "--- Setup Julia packages"
-      julia --color=yes --project=test -e '
-          using Pkg
-          Pkg.develop(path="lib/MPIPreferences")
-          using MPIPreferences
-          MPIPreferences.use_system_binary(export_prefs=true)
-          rm("test/Manifest.toml")'
-      julia -e 'import Pkg; Pkg.develop(; path = joinpath(pwd(), "lib", "MPIPreferences"))'
-      julia -e 'import Pkg; Pkg.develop(; path = pwd())'
-      julia -e 'import Pkg; Pkg.precompile()'
-
-      echo "+++ Run tests"
-      julia -e 'import Pkg; Pkg.test("MPI")'
+        echo "+++ Run tests"
+        julia --color=yes --project=. -e '
+           import Pkg
+           Pkg.test("MPI")
+           '
diff --git a/README.md b/README.md
index d17df0ea7..0deb644b3 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 [![Docs latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://juliaparallel.github.io/MPI.jl/latest/)
 [![Docs stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliaparallel.github.io/MPI.jl/stable/)
 [![Unit tests](https://github.com/JuliaParallel/MPI.jl/workflows/Unit%20Tests/badge.svg?branch=master)](https://github.com/JuliaParallel/MPI.jl/actions?query=workflow%3A%22Unit+Tests%22+branch%3Amaster)
-[![GPU tests](https://gitlab.com/JuliaGPU/MPI.jl/badges/master/pipeline.svg)](https://gitlab.com/JuliaGPU/MPI.jl/pipelines)
+[![GPU tests](https://badge.buildkite.com/ed813bc4d79f557adbdb821b1c8c8de98999686e697df4a373.svg?branch=master)](https://buildkite.com/julialang/mpi-dot-jl)
 [![codecov.io](https://codecov.io/github/JuliaParallel/MPI.jl/coverage.svg?branch=master)](https://codecov.io/github/JuliaParallel/MPI.jl?branch=master)
 [![Coverage Status](https://coveralls.io/repos/JuliaParallel/MPI.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/JuliaParallel/MPI.jl?branch=master)
 
diff --git a/test/common.jl b/test/common.jl
new file mode 100644
index 000000000..e583d5eb7
--- /dev/null
+++ b/test/common.jl
@@ -0,0 +1,11 @@
+using Test
+using MPI
+
+if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
+    import CUDA
+    ArrayType = CUDA.CuArray
+    synchronize() = CUDA.synchronize()
+else
+    ArrayType = Array
+    synchronize() = nothing
+end
diff --git a/test/test_allgather.jl b/test/test_allgather.jl
index ceece8501..0b6ada709 100644
--- a/test/test_allgather.jl
+++ b/test/test_allgather.jl
@@ -1,13 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
-
+include("common.jl")
 
 MPI.Init()
 
@@ -18,6 +9,7 @@ rank = MPI.Comm_rank(comm)
 for T in Base.uniontypes(MPI.MPIDatatype)
     # test vector input
     A = ArrayType{T}([rank + 1])
+    synchronize()
     C = MPI.Allgather(A, comm)
     @test C isa ArrayType{T,1}
     @test C == ArrayType{T,1}(1:size)
@@ -32,11 +24,12 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Test passing output buffer with set size
     A = ArrayType(T[val])
-    
+    synchronize()
+
     C = ArrayType{T}(undef, size)
     MPI.Allgather!(A, C, comm) # implied size
     @test C == ArrayType{T}(1:size)
-    
+
     C = ArrayType{T}(undef, size)
     MPI.Allgather!(A, UBuffer(C,1), comm)
     @test C == ArrayType{T}(1:size)
@@ -47,12 +40,14 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Test explicit IN_PLACE
     C = ArrayType{T}([i == rank ? i : size + 1 for i = 0:size-1])
+    synchronize()
     MPI.Allgather!(MPI.IN_PLACE, UBuffer(C, 1), comm)
     @test C isa ArrayType{T,1}
     @test C == ArrayType{T}(0:size-1)
 
     # Test IN_PLACE
     C = ArrayType{T}([i == rank ? i : size + 1 for i = 0:size-1])
+    synchronize()
     MPI.Allgather!(UBuffer(C, 1), comm)
     @test C isa ArrayType{T,1}
     @test C == ArrayType{T}(0:size-1)
diff --git a/test/test_allgatherv.jl b/test/test_allgatherv.jl
index e8fa2c5fd..22054444e 100644
--- a/test/test_allgatherv.jl
+++ b/test/test_allgatherv.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -19,6 +11,7 @@ check = collect(Iterators.flatten([fill(r, counts[r+1]) for r = 0:size-1]))
 
 for T in Base.uniontypes(MPI.MPIDatatype)
     A = ArrayType{T}(fill(T(rank), counts[rank+1]))
+    synchronize()
 
     # Test passing the output buffer
     B = ArrayType{T}(undef, sum(counts))
@@ -31,6 +24,7 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Test explicit MPI_IN_PLACE
     B = ArrayType(fill(T(rank), sum(counts)))
+    synchronize()
     MPI.Allgatherv!(MPI.IN_PLACE, VBuffer(B, counts), comm)
     @test B == ArrayType{T}(check)
 end
diff --git a/test/test_allreduce.jl b/test/test_allreduce.jl
index 63cb8548c..2404c4dd3 100644
--- a/test/test_allreduce.jl
+++ b/test/test_allreduce.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -19,12 +11,13 @@ if ArrayType != Array ||
     operators = [MPI.SUM, +]
 else
     operators = [MPI.SUM, +, (x,y) -> 2x+y-x]
-end    
+end
 
 for T = [Int]
     for dims = [1, 2, 3]
         send_arr = ArrayType(zeros(T, Tuple(3 for i in 1:dims)))
         send_arr[:] .= 1:length(send_arr)
+        synchronize()
 
         for op in operators
 
@@ -39,9 +32,10 @@ for T = [Int]
                                                        op, MPI.COMM_WORLD)
             # IN_PLACE
             recv_arr = copy(send_arr)
+            synchronize()
             MPI.Allreduce!(recv_arr, op, MPI.COMM_WORLD)
             @test recv_arr == comm_size .* send_arr
-            
+
             # Allocating version
             val = MPI.Allreduce(2, op, MPI.COMM_WORLD)
             @test val == comm_size*2
diff --git a/test/test_alltoall.jl b/test/test_alltoall.jl
index 41f89bb46..41e6dcff0 100644
--- a/test/test_alltoall.jl
+++ b/test/test_alltoall.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -18,23 +10,27 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Allocating version
     a = ArrayType(fill(T(rank), size))
+    synchronize()
     b = MPI.Alltoall(UBuffer(a, 1), comm)
     @test b isa ArrayType{T}
     @test b == ArrayType{T}(0:size-1)
 
     # Non Allocating version
     a = ArrayType(fill(T(rank),size))
+    synchronize()
     b = ArrayType{T}(undef, size*1)
     MPI.Alltoall!(UBuffer(a,1), UBuffer(b,1), comm)
     @test b == ArrayType{T}(0:size-1)
 
     # IN_PLACE version
     a = ArrayType{T}(fill(T(rank),size))
+    synchronize()
     MPI.Alltoall!(MPI.IN_PLACE, UBuffer(a,1), comm)
     @test a == ArrayType{T}(0:size-1)
 
     # One-argument IN_PLACE version
     a = ArrayType{T}(fill(T(rank),size))
+    synchronize()
     MPI.Alltoall!(UBuffer(a,1), comm)
     @test a == ArrayType{T}(0:size-1)
 end
diff --git a/test/test_alltoallv.jl b/test/test_alltoallv.jl
index c7996385a..84c5fc35e 100644
--- a/test/test_alltoallv.jl
+++ b/test/test_alltoallv.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -21,8 +13,9 @@ send_vals = collect(Iterators.flatten([1:i for i = 1:size]))
 recv_vals = collect(Iterators.flatten([1:rank+1 for i = 1:size]))
 
 for T in Base.uniontypes(MPI.MPIDatatype)
-        
+
     A = ArrayType{T}(send_vals)
+    synchronize()
 
     # Non Allocating version
     C = ArrayType{T}(undef, sum(recv_counts))
diff --git a/test/test_basic.jl b/test/test_basic.jl
index e9326c4ec..a2429b7aa 100644
--- a/test/test_basic.jl
+++ b/test/test_basic.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 @test !MPI.Initialized()
 MPI.Init()
diff --git a/test/test_bcast.jl b/test/test_bcast.jl
index 3bb374294..2c81cc747 100644
--- a/test/test_bcast.jl
+++ b/test/test_bcast.jl
@@ -1,14 +1,6 @@
-using Test
-using MPI
+include("common.jl")
 using Random
 
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
-
 MPI.Init()
 
 
diff --git a/test/test_exscan.jl b/test/test_exscan.jl
index a3b675447..3ca417dc2 100644
--- a/test/test_exscan.jl
+++ b/test/test_exscan.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -18,12 +10,13 @@ prodrank = prod(1:rank)
 # Not possible to PROD a Char (and neither Int8 with OpenMPI)
 for T in setdiff([Base.uniontypes(MPI.MPIDatatype)...], [Char, Int8, UInt8])
     A = ArrayType{T}(fill(T(rank+1), 4))
+    synchronize()
     B = similar(A)
     MPI.Exscan!(A, B, *, comm)
     if rank > 0
         @test B == ArrayType{T}(fill(T(prodrank), 4))
     end
-    
+
     B = MPI.Exscan(A, *, comm)
     @test B isa ArrayType{T}
 
@@ -33,11 +26,12 @@ for T in setdiff([Base.uniontypes(MPI.MPIDatatype)...], [Char, Int8, UInt8])
     end
 
     A = ArrayType{T}(fill(T(rank+1), 4))
+    synchronize()
     MPI.Exscan!(A, *, comm)
     if rank > 0
         @test A == ArrayType{T}(fill(T(prodrank), 4))
     end
-    
+
     B = MPI.Exscan(T(rank+1), *, comm)
     if rank > 0
         @test B[1] == prodrank
diff --git a/test/test_gather.jl b/test/test_gather.jl
index e55b8e0bd..489a6972a 100644
--- a/test/test_gather.jl
+++ b/test/test_gather.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -20,6 +12,7 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Allocating
     A = ArrayType{T}(fill(T(rank+1), 4))
+    synchronize()
     C = MPI.Gather(A, comm; root=root)
     if isroot
         @test C isa ArrayType{T}
@@ -37,6 +30,7 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Allocating, array
     A = ArrayType{T}([MPI.Comm_rank(comm) + 1])
+    synchronize()
     C = MPI.Gather(A, comm; root=root)
     if isroot
         @test C isa ArrayType{T}
@@ -47,6 +41,7 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Allocating, view
     A = ArrayType{T}([MPI.Comm_rank(comm) + 1, 0])
+    synchronize()
     C = MPI.Gather(view(A, 1:1), comm; root=root)
     if isroot
         @test C isa ArrayType{T}
@@ -55,6 +50,7 @@ for T in Base.uniontypes(MPI.MPIDatatype)
 
     # Non Allocating
     A = ArrayType{T}(fill(T(rank+1), 4))
+    synchronize()
     C = ArrayType{T}(undef, 4sz)
     MPI.Gather!(A, C, comm; root=root)
     if isroot
@@ -70,6 +66,7 @@ for T in Base.uniontypes(MPI.MPIDatatype)
     if isroot
         A = ArrayType{T}(fill(T(rank+1), 4*sz))
     end
+    synchronize()
     if root == MPI.Comm_rank(comm)
         MPI.Gather!(MPI.IN_PLACE, UBuffer(A,4), comm; root=root)
     else
diff --git a/test/test_gatherv.jl b/test/test_gatherv.jl
index 44ee44b0b..e3f196fd5 100644
--- a/test/test_gatherv.jl
+++ b/test/test_gatherv.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -21,6 +13,7 @@ check = collect(Iterators.flatten([fill(r, counts[r+1]) for r = 0:size-1]))
 
 for T in Base.uniontypes(MPI.MPIDatatype)
     A = ArrayType(fill(T(rank), mod(rank,2) + 1))
+    synchronize()
 
     # Test passing the output buffer
     B = ArrayType{T}(undef, sum(counts))
@@ -38,9 +31,11 @@ for T in Base.uniontypes(MPI.MPIDatatype)
     # Test explicit MPI_IN_PLACE
     if isroot
         B = ArrayType(fill(T(rank), sum(counts)))
+        synchronize()
         MPI.Gatherv!(MPI.IN_PLACE, VBuffer(B, counts), comm; root=root)
     else
         B = ArrayType(fill(T(rank), counts[rank+1]))
+        synchronize()
         MPI.Gatherv!(B, nothing, comm; root=root)
     end
     if isroot
diff --git a/test/test_io.jl b/test/test_io.jl
index 915ed116a..23f3bd589 100644
--- a/test/test_io.jl
+++ b/test/test_io.jl
@@ -1,14 +1,6 @@
-using Test
-using MPI
+include("common.jl")
 using Random
 
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
-
 MPI.Init()
 comm = MPI.COMM_WORLD
 rank = MPI.Comm_rank(comm)
diff --git a/test/test_io_shared.jl b/test/test_io_shared.jl
index 621f020de..b20bdc3d8 100644
--- a/test/test_io_shared.jl
+++ b/test/test_io_shared.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 comm = MPI.COMM_WORLD
diff --git a/test/test_io_subarray.jl b/test/test_io_subarray.jl
index 3f8f64c8d..5fa2323c9 100644
--- a/test/test_io_subarray.jl
+++ b/test/test_io_subarray.jl
@@ -1,13 +1,6 @@
-using Test
-using MPI
-using Random
+include("common.jl")
 
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+using Random
 
 MPI.Init()
 comm = MPI.COMM_WORLD
diff --git a/test/test_reduce.jl b/test/test_reduce.jl
index 396785204..3b3a4f927 100644
--- a/test/test_reduce.jl
+++ b/test/test_reduce.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 # Closures might not be supported by cfunction
 const can_do_closures =
diff --git a/test/test_scan.jl b/test/test_scan.jl
index 20a6924aa..1b114247e 100644
--- a/test/test_scan.jl
+++ b/test/test_scan.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -21,11 +13,11 @@ for T in setdiff([Base.uniontypes(MPI.MPIDatatype)...], [Char, Int8, UInt8])
     B = similar(A)
     MPI.Scan!(A, B, *, comm)
     @test B == ArrayType{T}(fill(T(prodrank), 4))
-    
+
     B = MPI.Scan(A, *, comm)
     @test B isa ArrayType{T}
     @test B == ArrayType{T}(fill(T(prodrank), 4))
-    
+
     B = MPI.Scan(T(rank+1), *, comm)
     @test B[1] == prodrank
 end
diff --git a/test/test_scatter.jl b/test/test_scatter.jl
index d55f7e09d..e3dc15cd9 100644
--- a/test/test_scatter.jl
+++ b/test/test_scatter.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
diff --git a/test/test_scatterv.jl b/test/test_scatterv.jl
index d30d1b475..6b601938b 100644
--- a/test/test_scatterv.jl
+++ b/test/test_scatterv.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
diff --git a/test/test_sendrecv.jl b/test/test_sendrecv.jl
index 22aea9ee7..45ad95c7b 100644
--- a/test/test_sendrecv.jl
+++ b/test/test_sendrecv.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
diff --git a/test/test_subarray.jl b/test/test_subarray.jl
index f197c8eea..fff3515d9 100644
--- a/test/test_subarray.jl
+++ b/test/test_subarray.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -19,6 +11,7 @@ src  = mod(rank-1, comm_size)
 
 @testset "contiguous" begin
     X = ArrayType(rank .+ collect(reshape(1.0:16.0, 4, 4)))
+    synchronize()
     Y = ArrayType(zeros(4))
     req_send = MPI.Isend(@view(X[:,1]), comm; dest=dest)
     req_recv = MPI.Irecv!(Y, comm; source=src)
@@ -28,6 +21,7 @@ src  = mod(rank-1, comm_size)
     @test Y == X[:,1] .- rank .+ src
 
     Y = ArrayType(zeros(2))
+    synchronize()
 
     req_send = MPI.Isend(Y, comm; dest=dest, tag=1)
     req_recv = MPI.Irecv!(@view(X[3:4,1]), comm; source=src, tag=1)
@@ -39,6 +33,7 @@ end
 
 @testset "strided" begin
     X = ArrayType(rank .+ collect(reshape(1.0:16.0, 4, 4)))
+    synchronize()
     Y = ArrayType(zeros(4))
     req_send = MPI.Isend(@view(X[2,:]), comm; dest=dest, tag=0)
     req_recv = MPI.Irecv!(Y, comm; source=src, tag=0)
@@ -48,6 +43,7 @@ end
     @test Y == X[2,:] .- rank .+ src
 
     Y = ArrayType(zeros(2))
+    synchronize()
 
     req_send = MPI.Isend(Y, comm; dest=dest, tag=1)
     req_recv = MPI.Irecv!(@view(X[3,1:2]), comm; source=src, tag=1)
@@ -59,6 +55,7 @@ end
 
 @testset "dense subarray" begin
     X = ArrayType(rank .+ collect(reshape(1.0:16.0, 4, 4)))
+    synchronize()
     Y = ArrayType(zeros(2,2))
     req_send = MPI.Isend(@view(X[2:3,3:4]), comm; dest=dest, tag=0)
     req_recv = MPI.Irecv!(Y, comm; source=src, tag=0)
@@ -68,6 +65,7 @@ end
     @test Y == X[2:3,3:4] .- rank .+ src
 
     Y = ArrayType(zeros(2,2))
+    synchronize()
 
     req_send = MPI.Isend(Y, comm; dest=dest, tag=1)
     req_recv = MPI.Irecv!(@view(X[3:4,1:2]), comm; source=src, tag=1)
@@ -77,6 +75,5 @@ end
     @test X[3:4,1:2] == Y
 end
 
-GC.gc()
 MPI.Finalize()
 @test MPI.Finalized()
diff --git a/test/test_test.jl b/test/test_test.jl
index ebaa057a9..e7d982923 100644
--- a/test/test_test.jl
+++ b/test/test_test.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -24,6 +16,7 @@ recv_mesg = ArrayType{Float64}(undef,N)
 recv_mesg_expected = ArrayType{Float64}(undef,N)
 fill!(send_mesg, Float64(rank))
 fill!(recv_mesg_expected, Float64(src))
+synchronize()
 
 rreq = MPI.Irecv!(recv_mesg, comm; source=src, tag=src+32)
 sreq = MPI.Isend(send_mesg, comm; dest=dst, tag=rank+32, )
diff --git a/test/test_threads.jl b/test/test_threads.jl
index 90ecbf40b..9a7592ed2 100644
--- a/test/test_threads.jl
+++ b/test/test_threads.jl
@@ -1,12 +1,4 @@
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 provided = MPI.Init(threadlevel=:multiple)
 
diff --git a/test/test_wait.jl b/test/test_wait.jl
index 7d292e055..c3521d115 100644
--- a/test/test_wait.jl
+++ b/test/test_wait.jl
@@ -1,13 +1,5 @@
 # tests for the various kinds of waits
-using Test
-using MPI
-
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
-    import CUDA
-    ArrayType = CUDA.CuArray
-else
-    ArrayType = Array
-end
+include("common.jl")
 
 MPI.Init()
 
@@ -17,6 +9,7 @@ commsize = MPI.Comm_rank(MPI.COMM_WORLD)
 nsends = 2
 send_arr = [ArrayType{Int}([i]) for i = 1:nsends]
 recv_arr = [ArrayType{Int}(undef,1) for i = 1:nsends]
+synchronize()
 
 # send to self
 send_reqs = [MPI.Isend(send_arr[i], MPI.COMM_WORLD; dest=myrank, tag=i) for i = 1:nsends]