From 5227a1369321791687e35ce9925f33d0a9643075 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 08:46:15 +0100 Subject: [PATCH 01/48] fix pipeline --- .buildkite/distributed/pipeline.yml | 18 +++++++++++++++--- test/utils_for_runtests.jl | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index fe38f7bea1..2a0f00841a 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -17,10 +17,18 @@ steps: env: TEST_GROUP: "init" command: - - echo "--- Instantiate project" - - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" + - "echo '--- Instantiate project'" + - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + + - "echo '--- Precompile project'" + - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "julia --project -e 'using Pkg; Pkg.status()'" + + # Force the initialization of the CUDA runtime as it is lazily loaded by default: + - "echo '--- Initialize the CUDA runtime'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "julia --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G slurm_gpus: 1 slurm_cpus_per_task: 8 @@ -40,6 +48,7 @@ steps: key: "distributed_gpu" env: TEST_GROUP: "distributed" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -66,6 +75,7 @@ steps: key: "distributed_solvers_gpu" env: TEST_GROUP: "distributed_solvers" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -91,6 +101,7 @@ steps: key: "distributed_hydrostatic_model_gpu" env: TEST_GROUP: "distributed_hydrostatic_model" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -116,6 +127,7 @@ steps: key: "distributed_nonhydrostatic_regression_gpu" env: TEST_GROUP: "distributed_nonhydrostatic_regression" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index 7bc06e528e..298f68db16 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -3,7 +3,7 @@ using Oceananigans.DistributedComputations: Distributed, Partition, child_archit import Oceananigans.Fields: interior -test_child_arch() = CUDA.has_cuda() ? GPU() : CPU() +test_child_arch() = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() function test_architectures() child_arch = test_child_arch() From 1225061aa8fa9d20fe851e4de804ea30b4f07567 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 08:53:10 +0100 Subject: [PATCH 02/48] mpi test and gpu test --- .buildkite/distributed/pipeline.yml | 8 ++++++++ test/utils_for_runtests.jl | 31 +++++++++++++++++++---------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 2a0f00841a..3ebd4de52c 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -38,6 +38,7 @@ steps: key: "distributed_cpu" env: TEST_GROUP: "distributed" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -49,6 +50,7 @@ steps: env: TEST_GROUP: "distributed" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -65,6 +67,7 @@ steps: key: "distributed_solvers_cpu" env: TEST_GROUP: "distributed_solvers" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -76,6 +79,7 @@ steps: env: TEST_GROUP: "distributed_solvers" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -91,6 +95,7 @@ steps: key: "distributed_hydrostatic_model_cpu" env: TEST_GROUP: "distributed_hydrostatic_model" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -102,6 +107,7 @@ steps: env: TEST_GROUP: "distributed_hydrostatic_model" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -117,6 +123,7 @@ steps: key: "distributed_nonhydrostatic_regression_cpu" env: TEST_GROUP: "distributed_nonhydrostatic_regression" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -128,6 +135,7 @@ steps: env: TEST_GROUP: "distributed_nonhydrostatic_regression" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index 298f68db16..e8741e4839 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -4,6 +4,7 @@ using Oceananigans.DistributedComputations: Distributed, Partition, child_archit import Oceananigans.Fields: interior test_child_arch() = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() +mpi_test() = parse(Bool, get(ENV, "MPI_TEST", "false")) function test_architectures() child_arch = test_child_arch() @@ -11,13 +12,17 @@ function test_architectures() # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test several different configurations: `Partition(x = 4)`, `Partition(y = 4)`, # `Partition(x = 2, y = 2)`, and different fractional subdivisions in x, y and xy - if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 - return (Distributed(child_arch; partition = Partition(4)), - Distributed(child_arch; partition = Partition(1, 4)), - Distributed(child_arch; partition = Partition(2, 2)), - Distributed(child_arch; partition = Partition(x = Fractional(1, 2, 3, 4))), - Distributed(child_arch; partition = Partition(y = Fractional(1, 2, 3, 4))), - Distributed(child_arch; partition = Partition(x = Fractional(1, 2), y = Equal()))) + if mpi_test() + if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 + return (Distributed(child_arch; partition = Partition(4)), + Distributed(child_arch; partition = Partition(1, 4)), + Distributed(child_arch; partition = Partition(2, 2)), + Distributed(child_arch; partition = Partition(x = Fractional(1, 2, 3, 4))), + Distributed(child_arch; partition = Partition(y = Fractional(1, 2, 3, 4))), + Distributed(child_arch; partition = Partition(x = Fractional(1, 2), y = Equal()))) + else + return throw("The MPI partitioning is not correctly configured.") + end else return tuple(child_arch) end @@ -31,10 +36,14 @@ function nonhydrostatic_regression_test_architectures() # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test 3 different configurations: `Partition(x = 4)`, `Partition(y = 4)` # and `Partition(x = 2, y = 2)` - if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 - return (Distributed(child_arch; partition = Partition(4)), - Distributed(child_arch; partition = Partition(1, 4)), - Distributed(child_arch; partition = Partition(2, 2))) + if mpi_test() + if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 + return (Distributed(child_arch; partition = Partition(4)), + Distributed(child_arch; partition = Partition(1, 4)), + Distributed(child_arch; partition = Partition(2, 2))) + else + return throw("The MPI partitioning is not correctly configured.") + end else return tuple(child_arch) end From 1652c6bd8948c7d53e7ea580068395769716f6ff Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 11:22:17 +0100 Subject: [PATCH 03/48] do we need to precompile it inside? --- .buildkite/distributed/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 3ebd4de52c..eb9b58bd5a 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -52,6 +52,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "julia -O0 --color=yes --project -e 'using CUDA; CUDA.precompile_runtime()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From 93232037fc96a05f92b7294c8c0d843fe6442b49 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 11:49:55 +0100 Subject: [PATCH 04/48] precompile inside the node --- .buildkite/distributed/pipeline.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index eb9b58bd5a..77731654a9 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -18,18 +18,20 @@ steps: TEST_GROUP: "init" command: - "echo '--- Instantiate project'" - - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + - "srun julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - "echo '--- Precompile project'" - - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" - - "julia --project -e 'using Pkg; Pkg.status()'" + - "srun julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "srun julia --project -e 'using Pkg; Pkg.status()'" # Force the initialization of the CUDA runtime as it is lazily loaded by default: - "echo '--- Initialize the CUDA runtime'" - - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - - "julia --project -e 'using Pkg; Pkg.test()'" + - "srun julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "srun julia --project -e 'using Pkg; Pkg.test()'" agents: + slurm_ntasks: 1 slurm_gpus: 1 + slurm_gpus_per_task: 1 slurm_cpus_per_task: 8 - wait @@ -52,7 +54,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --color=yes --project -e 'using CUDA; CUDA.precompile_runtime()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From 37b17ff714adab1d76b434e8619a2e035d4da42c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 11:51:39 +0100 Subject: [PATCH 05/48] try previous climacommon version --- .buildkite/distributed/pipeline.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 77731654a9..5187915441 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,7 +1,7 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_10_09 + modules: climacommon/2024_05_27 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" @@ -18,20 +18,18 @@ steps: TEST_GROUP: "init" command: - "echo '--- Instantiate project'" - - "srun julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - "echo '--- Precompile project'" - - "srun julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" - - "srun julia --project -e 'using Pkg; Pkg.status()'" + - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "julia --project -e 'using Pkg; Pkg.status()'" # Force the initialization of the CUDA runtime as it is lazily loaded by default: - "echo '--- Initialize the CUDA runtime'" - - "srun julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - - "srun julia --project -e 'using Pkg; Pkg.test()'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "julia --project -e 'using Pkg; Pkg.test()'" agents: - slurm_ntasks: 1 slurm_gpus: 1 - slurm_gpus_per_task: 1 slurm_cpus_per_task: 8 - wait From 2ac8cdedc51183633fd174f3c2cdb0c77c428ce9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 12:33:30 +0100 Subject: [PATCH 06/48] go even more back --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 5187915441..c01f6560e5 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,7 +1,7 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_05_27 + modules: climacommon/2024_02_27 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" From 0eb2720e2822234f6da1b75c89b593a5fb5fa59b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 13:19:05 +0100 Subject: [PATCH 07/48] use the ClimaOcean implementation --- .buildkite/distributed/pipeline.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index c01f6560e5..7787aa351e 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,19 +1,18 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_02_27 + modules: climacommon/2024_05_27 + +timeout_in_minutes: 1440 env: - JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 + JULIA_NUM_PRECOMPILE_TASKS: 8 steps: - label: "initialize" - key: "init_central" + key: "init" env: TEST_GROUP: "init" command: From 50d0ec3c08c6a8e2c2ea2d7d0728773bbe0dd34f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 13:19:20 +0100 Subject: [PATCH 08/48] using the ClimaOcean implementation --- .buildkite/distributed/pipeline.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 7787aa351e..20753bc6f1 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -3,8 +3,6 @@ agents: slurm_mem: 8G modules: climacommon/2024_05_27 -timeout_in_minutes: 1440 - env: OPENBLAS_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 From 6e183bd3161a42b3cfcd25a34b772e3c13ddd145 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 13:39:41 +0100 Subject: [PATCH 09/48] see if this test passes --- .buildkite/distributed/pipeline.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 20753bc6f1..57859f199d 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -49,16 +49,21 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "echo '--- Instantiate project'" + - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + + - "echo '--- Precompile project'" + - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "julia --project -e 'using Pkg; Pkg.status()'" + + # Force the initialization of the CUDA runtime as it is lazily loaded by default: + - "echo '--- Initialize the CUDA runtime'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G slurm_ntasks: 4 slurm_gpus_per_task: 1 - retry: - automatic: - - exit_status: 1 - limit: 1 - - label: "🦾 cpu distributed solvers tests" key: "distributed_solvers_cpu" From c56b15b16c1561a1bfa6ea43ca250cb3e2ad7f39 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 15:54:23 +0100 Subject: [PATCH 10/48] maybe precompiling before... --- .buildkite/distributed/pipeline.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 57859f199d..4d5a88ebc0 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -49,16 +49,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "echo '--- Instantiate project'" - - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - - - "echo '--- Precompile project'" - - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" - - "julia --project -e 'using Pkg; Pkg.status()'" - - # Force the initialization of the CUDA runtime as it is lazily loaded by default: - - "echo '--- Initialize the CUDA runtime'" - - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "julia -O0 -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From e30973feb600dda1564db9a093bdbba8da6c8dc8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 16:48:52 +0100 Subject: [PATCH 11/48] double O0 --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 4d5a88ebc0..ec12aad398 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -49,7 +49,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" + - "julia -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From e4cb16ecb53dbdf52f7721baec7f10b3f1f0902b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 17:46:25 +0100 Subject: [PATCH 12/48] back to previous clima_common --- .buildkite/distributed/pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index ec12aad398..0c0fb40d8d 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,7 +1,7 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_05_27 + modules: climacommon/2024_10_09 env: OPENBLAS_NUM_THREADS: 1 @@ -49,7 +49,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From 0c1f01c79a704fced0e52bad34ce027f8fb7d94d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 09:33:23 +0100 Subject: [PATCH 13/48] another quick test --- test/runtests.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 816d8d578a..11a4e553e8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -173,11 +173,13 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() + CUDA.set_runtime_version!(v"12.6") include("test_distributed_models.jl") end if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.6") include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end @@ -185,12 +187,14 @@ CUDA.allowscalar() do if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() + CUDA.set_runtime_version!(v"12.6") include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") end if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.6") archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From bec1cd1e0728ed6da36b9b3e80d4b408ec078eaf Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 14:40:18 +0100 Subject: [PATCH 14/48] change environment --- .buildkite/distributed/pipeline.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 0c0fb40d8d..c59078ca24 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -4,15 +4,18 @@ agents: modules: climacommon/2024_10_09 env: + JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - OMPI_MCA_opal_warn_on_missing_libcuda: 0 + JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager JULIA_NUM_PRECOMPILE_TASKS: 8 + JULIA_NUM_THREADS: 8 + OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: - label: "initialize" key: "init" env: - TEST_GROUP: "init" + TEST_GROUP: "init_central" command: - "echo '--- Instantiate project'" - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" @@ -26,6 +29,7 @@ steps: - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - "julia --project -e 'using Pkg; Pkg.test()'" agents: + slurm_mem: 120G slurm_gpus: 1 slurm_cpus_per_task: 8 From 75546af3166e5e9969c2d8fe3282d83395aa30b8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 14:42:40 +0100 Subject: [PATCH 15/48] correct the utils --- test/utils_for_runtests.jl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index e8741e4839..a64d44263b 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -3,16 +3,16 @@ using Oceananigans.DistributedComputations: Distributed, Partition, child_archit import Oceananigans.Fields: interior -test_child_arch() = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() -mpi_test() = parse(Bool, get(ENV, "MPI_TEST", "false")) +# Are the test running on the GPUs? +# Are the test running in parallel? +child_arch = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() +mpi_test = parse(Bool, get(ENV, "MPI_TEST", "false")) function test_architectures() - child_arch = test_child_arch() - # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test several different configurations: `Partition(x = 4)`, `Partition(y = 4)`, # `Partition(x = 2, y = 2)`, and different fractional subdivisions in x, y and xy - if mpi_test() + if mpi_test if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 return (Distributed(child_arch; partition = Partition(4)), Distributed(child_arch; partition = Partition(1, 4)), @@ -31,12 +31,10 @@ end # For nonhydrostatic simulations we cannot use `Fractional` at the moment (requirements # for the tranpose are more stringent than for hydrostatic simulations). function nonhydrostatic_regression_test_architectures() - child_arch = test_child_arch() - # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test 3 different configurations: `Partition(x = 4)`, `Partition(y = 4)` # and `Partition(x = 2, y = 2)` - if mpi_test() + if mpi_test if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 return (Distributed(child_arch; partition = Partition(4)), Distributed(child_arch; partition = Partition(1, 4)), From 9b334af5cf796ab4eb08d708374152c7dcb90bb8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 4 Nov 2024 15:25:24 +0100 Subject: [PATCH 16/48] this should load mpitrampoline --- .buildkite/distributed/pipeline.yml | 32 ++++++++---------------- test/test_distributed_poisson_solvers.jl | 4 +-- test/test_distributed_transpose.jl | 2 -- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index c59078ca24..0fcd204243 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,38 +1,28 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_10_09 + modules: climacommon/2024_05_27 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: - label: "initialize" - key: "init" - env: - TEST_GROUP: "init_central" + key: "init_central" command: - - "echo '--- Instantiate project'" - - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - - - "echo '--- Precompile project'" - - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - echo "--- Instantiate project" + - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'" + # force the initialization of the CUDA runtime as it is lazily loaded by default + - "julia --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" + - "julia --project -e 'using MPI; MPI.versioninfo()'" + # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl + - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" + + - echo "--- Instantiate status" - "julia --project -e 'using Pkg; Pkg.status()'" - # Force the initialization of the CUDA runtime as it is lazily loaded by default: - - "echo '--- Initialize the CUDA runtime'" - - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - - "julia --project -e 'using Pkg; Pkg.test()'" - agents: - slurm_mem: 120G - slurm_gpus: 1 - slurm_cpus_per_task: 8 - - wait - label: "🐉 cpu distributed unit tests" diff --git a/test/test_distributed_poisson_solvers.jl b/test/test_distributed_poisson_solvers.jl index dca406d8c3..5ab7c90a7e 100644 --- a/test/test_distributed_poisson_solvers.jl +++ b/test/test_distributed_poisson_solvers.jl @@ -121,9 +121,7 @@ function divergence_free_poisson_tridiagonal_solution(grid_points, ranks, stretc return Array(interior(∇²ϕ)) ≈ Array(R) end -@testset "Distributed FFT-based Poisson solver" begin - child_arch = test_child_arch() - +@testset "Distributed FFT-based Poisson solver" begin for topology in ((Periodic, Periodic, Periodic), (Periodic, Periodic, Bounded), (Periodic, Bounded, Bounded), diff --git a/test/test_distributed_transpose.jl b/test/test_distributed_transpose.jl index 5bc0fe6292..2eff450b27 100644 --- a/test/test_distributed_transpose.jl +++ b/test/test_distributed_transpose.jl @@ -38,8 +38,6 @@ function test_transpose(grid_points, ranks, topo, child_arch) end @testset "Distributed Transpose" begin - child_arch = test_child_arch() - for topology in ((Periodic, Periodic, Periodic), (Periodic, Periodic, Bounded), (Periodic, Bounded, Bounded), From f8c64016b94e0758f05d0cef9d7e641a3d6bae14 Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Mon, 4 Nov 2024 15:03:56 -0700 Subject: [PATCH 17/48] Fix formatting --- .buildkite/distributed/pipeline.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 0fcd204243..02223bf2dc 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -13,10 +13,12 @@ steps: key: "init_central" command: - echo "--- Instantiate project" - - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'" - # force the initialization of the CUDA runtime as it is lazily loaded by default + - "julia --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" + + # Force the initialization of the CUDA runtime as it is lazily loaded by default - "julia --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "julia --project -e 'using MPI; MPI.versioninfo()'" + # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" From 1dc42bba5b2e15ec24eb5f8916bb122c0d0b99eb Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Mon, 4 Nov 2024 15:24:56 -0700 Subject: [PATCH 18/48] Go back to latest climacommon --- .buildkite/distributed/pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 02223bf2dc..e0ea62eef4 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,10 +1,11 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_05_27 + modules: climacommon/2024_10_08 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" + JULIA_MPI_HAS_CUDA: "true" OPENBLAS_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 From 5a870e787b76e40e2ba8bb4d1ec2d81ee1b71bb2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 09:54:12 +0100 Subject: [PATCH 19/48] try adding Manifest --- Manifest.toml | 972 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 972 insertions(+) create mode 100644 Manifest.toml diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000000..35f1827b15 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,972 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.1" +manifest_format = "2.0" +project_hash = "f3d268da10603d34af6d3fea2014bfc2339ad299" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + + [deps.AbstractFFTs.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "50c3c56a52972d78e8be9fd135bfb91c9574c140" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "4.1.1" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "0.1.0" + +[[deps.BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random", "Test"] +git-tree-sha1 = "2c7cc21e8678eff479978a0a2ef5ce2f51b63dff" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.5.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.Blosc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "ef12cdd1c7fb7e1dfd6fa8fd60d4db6bc61d2f23" +uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9" +version = "1.21.6+0" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "8873e196c2eb87962a2048b3b8e08946535864a1" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+2" + +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + +[[deps.CFTime]] +deps = ["Dates", "Printf"] +git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1" +uuid = "179af706-886a-5703-950a-314cd64e0468" +version = "0.1.3" + +[[deps.CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "StaticArrays", "Statistics", "demumble_jll"] +git-tree-sha1 = "e0725a467822697171af4dae15cec10b4fc19053" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "5.5.2" + + [deps.CUDA.extensions] + ChainRulesCoreExt = "ChainRulesCore" + EnzymeCoreExt = "EnzymeCore" + SpecialFunctionsExt = "SpecialFunctions" + + [deps.CUDA.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" + +[[deps.CUDA_Driver_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ccd1e54610c222fadfd4737dac66bff786f63656" +uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" +version = "0.10.3+0" + +[[deps.CUDA_Runtime_Discovery]] +deps = ["Libdl"] +git-tree-sha1 = "33576c7c1b2500f8e7e6baa082e04563203b3a45" +uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" +version = "0.3.5" + +[[deps.CUDA_Runtime_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "e43727b237b2879a34391eeb81887699a26f8f2f" +uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +version = "0.15.3+0" + +[[deps.ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.5" + +[[deps.Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.11" + +[[deps.CommonDataModel]] +deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] +git-tree-sha1 = "d6fb5bf939a2753c74984b11434ea25d6c397a58" +uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" +version = "0.3.6" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.16.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.ConstructionBase]] +git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.8" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.CubedSphere]] +deps = ["TaylorSeries"] +git-tree-sha1 = "51bb25de518b4c62b7cdf26e5fbb84601bb27a60" +uuid = "7445602f-e544-4518-8976-18f8e8ae6cdb" +version = "0.3.0" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.7.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DiskArrays]] +deps = ["LRUCache", "OffsetArrays"] +git-tree-sha1 = "e0e89a60637a62d13aa2107f0acd169b9b9b77e7" +uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +version = "0.4.6" + +[[deps.Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.12" + + [deps.Distances.extensions] + DistancesChainRulesCoreExt = "ChainRulesCore" + DistancesSparseArraysExt = "SparseArrays" + + [deps.Distances.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.FFTW]] +deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] +git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" +uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +version = "1.8.0" + +[[deps.FFTW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4d81ed14783ec49ce9f2e168208a12ce1815aa25" +uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" +version = "3.3.10+1" + +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "62ca0547a14c57e98154423419d8a342dca75ca9" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.16.4" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.5" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GMP_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" +version = "6.3.0+0" + +[[deps.GPUArrays]] +deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] +git-tree-sha1 = "62ee71528cca49be797076a76bdc654a170a523e" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "10.3.1" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "ec632f177c0d990e64d955ccc1b8c04c485a0950" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.1.6" + +[[deps.GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "PrecompileTools", "Preferences", "Scratch", "Serialization", "TOML", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "1d6f290a5eb1201cd63574fbc4440c788d5cb38f" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.27.8" + +[[deps.Glob]] +git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" +uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" +version = "1.3.1" + +[[deps.GnuTLS_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Nettle_jll", "P11Kit_jll", "Zlib_jll"] +git-tree-sha1 = "383db7d3f900f4c1f47a8a04115b053c095e48d3" +uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" +version = "3.8.4+0" + +[[deps.HDF5_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] +git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" +uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" +version = "1.14.2+1" + +[[deps.Hwloc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "dd3b49277ec2bb2c6b94eb1604d4d0616016f7a6" +uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" +version = "2.11.2+0" + +[[deps.IncompleteLU]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "6c676e79f98abb6d33fa28122cad099f1e464afe" +uuid = "40713840-3770-5561-ab4c-a76e7d0d7895" +version = "0.2.1" + +[[deps.InlineStrings]] +git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.2" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + ParsersExt = "Parsers" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" + +[[deps.IntelOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] +git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e" +uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" +version = "2024.2.1+0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.InvertedIndices]] +git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.0" + +[[deps.IterativeSolvers]] +deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] +git-tree-sha1 = "59545b0a2b27208b0650df0a46b8e3019f85055b" +uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" +version = "0.9.4" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLD2]] +deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] +git-tree-sha1 = "783c1be5213a09609b23237a0c9e5dfd258ae6f2" +uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +version = "0.5.7" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.6.1" + +[[deps.JuliaNVTXCallbacks_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "af433a10f3942e882d3c671aacb203e006a5808f" +uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e" +version = "0.2.1+0" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "e73a077abc7fe798fe940deabe30ef6c66bdde52" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.29" + + [deps.KernelAbstractions.extensions] + EnzymeExt = "EnzymeCore" + LinearAlgebraExt = "LinearAlgebra" + SparseArraysExt = "SparseArrays" + + [deps.KernelAbstractions.weakdeps] + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Unicode"] +git-tree-sha1 = "d422dfd9707bec6617335dc2ea3c5172a87d5908" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "9.1.3" +weakdeps = ["BFloat16s"] + + [deps.LLVM.extensions] + BFloat16sExt = "BFloat16s" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "05a8bd5a42309a9ec82f700876903abce1017dd3" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.34+0" + +[[deps.LLVMLoopInfo]] +git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" +uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" +version = "1.0.0" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "18.1.7+0" + +[[deps.LRUCache]] +git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" +uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" +version = "1.6.1" +weakdeps = ["Serialization"] + + [deps.LRUCache.extensions] + SerializationExt = ["Serialization"] + +[[deps.LaTeXStrings]] +git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.4.0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+1" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.Lz4_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "abf88ff67f4fd89839efcae2f4c39cbc4ecd0846" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.10.0+1" + +[[deps.MKL_jll]] +deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] +git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f" +uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" +version = "2024.2.0+0" + +[[deps.MPI]] +deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "PkgVersion", "PrecompileTools", "Requires", "Serialization", "Sockets"] +git-tree-sha1 = "892676019c58f34e38743bc989b0eca5bce5edc5" +uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195" +version = "0.20.22" + + [deps.MPI.extensions] + AMDGPUExt = "AMDGPU" + CUDAExt = "CUDA" + + [deps.MPI.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + +[[deps.MPICH_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "7715e65c47ba3941c502bffb7f266a41a7f54423" +uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" +version = "4.2.3+0" + +[[deps.MPIPreferences]] +deps = ["Libdl", "Preferences"] +git-tree-sha1 = "c105fe467859e7f6e9a852cb15cb4301126fac07" +uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" +version = "0.1.11" + +[[deps.MPItrampoline_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "70e830dab5d0775183c99fc75e4c24c614ed7142" +uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" +version = "5.5.1+0" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.13" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.MicrosoftMPI_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" +uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" +version = "10.1.4+2" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.NCDatasets]] +deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] +git-tree-sha1 = "2c9dc92001ac06d432f363f37ff5552954d9947c" +uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" +version = "0.14.6" + +[[deps.NVTX]] +deps = ["Colors", "JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"] +git-tree-sha1 = "53046f0483375e3ed78e49190f1154fa0a4083a1" +uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f" +version = "0.3.4" + +[[deps.NVTX_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ce3269ed42816bf18d500c9f63418d4b0d9f5a3b" +uuid = "e98f9f5b-d649-5603-91fd-7774390e6439" +version = "3.1.0+2" + +[[deps.NetCDF_jll]] +deps = ["Artifacts", "Blosc_jll", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenMPI_jll", "XML2_jll", "Zlib_jll", "Zstd_jll", "libzip_jll"] +git-tree-sha1 = "a8af1798e4eb9ff768ce7fdefc0e957097793f15" +uuid = "7243133f-43d8-5620-bbf4-c2c921802cf3" +version = "400.902.209+0" + +[[deps.Nettle_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "eca63e3847dad608cfa6a3329b95ef674c7160b4" +uuid = "4c82536e-c426-54e4-b420-14f461c4ed8b" +version = "3.7.2+0" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OffsetArrays]] +git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.14.1" +weakdeps = ["Adapt"] + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.27+1" + +[[deps.OpenMPI_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] +git-tree-sha1 = "bfce6d523861a6c562721b262c0d1aaeead2647f" +uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" +version = "5.0.5+0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.P11Kit_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" +uuid = "c2071276-7c44-58a7-b746-946036e04d0a" +version = "0.24.1+0" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" + + [deps.Pkg.extensions] + REPLExt = "REPL" + + [deps.Pkg.weakdeps] + REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.PkgVersion]] +deps = ["Pkg"] +git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" +uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" +version = "0.3.3" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.4.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Quaternions]] +deps = ["LinearAlgebra", "Random", "RealDot"] +git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" +uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" +version = "0.7.6" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "4743b43e5a9c4a2ede372de7061eed81795b12e7" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.7.0" + +[[deps.RandomNumbers]] +deps = ["Random"] +git-tree-sha1 = "c6ec94d2aaba1ab2ff983052cf6a606ca5985902" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.6.0" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rotations]] +deps = ["LinearAlgebra", "Quaternions", "Random", "StaticArrays"] +git-tree-sha1 = "5680a9276685d392c87407df00d57c9924d9f11e" +uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" +version = "1.7.1" +weakdeps = ["RecipesBase"] + + [deps.Rotations.extensions] + RotationsRecipesBaseExt = "RecipesBase" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.2.1" + +[[deps.SeawaterPolynomials]] +git-tree-sha1 = "78f965a2f0cd5250a20c9aba9979346dd2b35734" +uuid = "d496a93d-167e-4197-9f49-d3af4ff8fe40" +version = "0.3.5" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "d0553ce4031a081cc42387a9b9c8441b7d99f32d" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.7" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.11.0" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "777657803913ffc7e8cc20f0fd04b634f871af8f" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.8" + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.3" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.7.0" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.4.0" + +[[deps.StructArrays]] +deps = ["ConstructionBase", "DataAPI", "Tables"] +git-tree-sha1 = "f4dc295e983502292c4c3f951dbb4e985e35b3be" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.18" +weakdeps = ["Adapt", "GPUArraysCore", "SparseArrays", "StaticArrays"] + + [deps.StructArrays.extensions] + StructArraysAdaptExt = "Adapt" + StructArraysGPUArraysCoreExt = "GPUArraysCore" + StructArraysSparseArraysExt = "SparseArrays" + StructArraysStaticArraysExt = "StaticArrays" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.7.0+0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.TaylorSeries]] +deps = ["LinearAlgebra", "Markdown", "Requires", "SparseArrays"] +git-tree-sha1 = "abc13c4d3cccd1703335ba03abdaf0dc8ecc97e2" +uuid = "6aa5eb33-94cf-58f4-a9d0-e4b2c4fc25ea" +version = "0.17.3" + + [deps.TaylorSeries.extensions] + TaylorSeriesIAExt = "IntervalArithmetic" + TaylorSeriesSAExt = "StaticArrays" + + [deps.TaylorSeries.weakdeps] + IntervalArithmetic = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "3a6f063d690135f5c1ba351412c82bae4d1402bf" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.25" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.2.1" + +[[deps.UnsafeAtomicsLLVM]] +deps = ["LLVM", "UnsafeAtomics"] +git-tree-sha1 = "2d17fabcd17e67d7625ce9c531fb9f40b7c42ce4" +uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" +version = "0.2.1" + +[[deps.XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] +git-tree-sha1 = "6a451c6f33a176150f315726eba8b92fbfdb9ae7" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.13.4+0" + +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "15e637a697345f6743674f1322beefbc5dcd5cfc" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.6.3+0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.6+1" + +[[deps.demumble_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6498e3581023f8e530f34760d18f75a69e3a4ea8" +uuid = "1e29f10c-031c-5a83-9565-69cddfc27673" +version = "1.3.0+0" + +[[deps.libaec_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" +uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" +version = "1.1.2+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.11.0+0" + +[[deps.libzip_jll]] +deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "668ac0297e6bd8f4d53dfdcd3ace71f2e00f4a35" +uuid = "337d8026-41b4-5cde-a456-74a10e5b31d1" +version = "1.11.1+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.oneTBB_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7d0ea0f4895ef2f5cb83645fa689e52cb55cf493" +uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" +version = "2021.12.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" From 9e63f564d4b4c2564d22ae979622dc0a927f1cf6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 10:03:33 +0100 Subject: [PATCH 20/48] Manifest from julia 1.10 --- Manifest.toml | 68 +++++++++++++++------------------------------------ 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 35f1827b15..45d969c32d 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.1" +julia_version = "1.10.6" manifest_format = "2.0" -project_hash = "f3d268da10603d34af6d3fea2014bfc2339ad299" +project_hash = "475d8d28ec1bf53471638e448deed724aee15ccd" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -30,11 +30,10 @@ weakdeps = ["StaticArrays"] [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.2" +version = "1.1.1" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -version = "1.11.0" [[deps.Atomix]] deps = ["UnsafeAtomics"] @@ -50,7 +49,6 @@ version = "0.5.0" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" [[deps.Blosc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] @@ -193,7 +191,6 @@ version = "1.0.0" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" [[deps.DiskArrays]] deps = ["LRUCache", "OffsetArrays"] @@ -218,7 +215,6 @@ version = "0.10.12" [[deps.Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -version = "1.11.0" [[deps.DocStringExtensions]] deps = ["LibGit2"] @@ -256,7 +252,6 @@ version = "1.16.4" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -version = "1.11.0" [[deps.FixedPointNumbers]] deps = ["Statistics"] @@ -267,12 +262,11 @@ version = "0.8.5" [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" -version = "1.11.0" [[deps.GMP_jll]] deps = ["Artifacts", "Libdl"] uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" -version = "6.3.0+0" +version = "6.2.1+6" [[deps.GPUArrays]] deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] @@ -343,7 +337,6 @@ version = "2024.2.1+0" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" [[deps.InvertedIndices]] git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" @@ -439,7 +432,6 @@ version = "1.4.0" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" -version = "1.11.0" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] @@ -449,17 +441,16 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.6.0+0" +version = "8.4.0+0" [[deps.LibGit2]] deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -version = "1.11.0" [[deps.LibGit2_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.7.2+0" +version = "1.6.4+0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] @@ -468,7 +459,6 @@ version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -version = "1.11.0" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -479,11 +469,9 @@ version = "1.17.0+1" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -version = "1.11.0" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" [[deps.Lz4_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -538,12 +526,11 @@ version = "0.5.13" [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.6+0" +version = "2.28.2+1" [[deps.MicrosoftMPI_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -559,11 +546,10 @@ version = "1.2.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.12.12" +version = "2023.1.10" [[deps.NCDatasets]] deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] @@ -611,7 +597,7 @@ weakdeps = ["Adapt"] [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.27+1" +version = "0.3.23+4" [[deps.OpenMPI_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] @@ -637,15 +623,9 @@ uuid = "c2071276-7c44-58a7-b746-946036e04d0a" version = "0.24.1+0" [[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.11.0" - - [deps.Pkg.extensions] - REPLExt = "REPL" - - [deps.Pkg.weakdeps] - REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.10.0" [[deps.PkgVersion]] deps = ["Pkg"] @@ -680,7 +660,6 @@ version = "2.4.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" [[deps.Quaternions]] deps = ["LinearAlgebra", "Random", "RealDot"] @@ -688,10 +667,13 @@ git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" version = "0.7.6" +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + [[deps.Random]] deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" [[deps.Random123]] deps = ["Random", "RandomNumbers"] @@ -761,11 +743,9 @@ version = "1.4.7" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -version = "1.11.0" [[deps.SortingAlgorithms]] deps = ["DataStructures"] @@ -776,7 +756,7 @@ version = "1.2.1" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.11.0" +version = "1.10.0" [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] @@ -798,14 +778,9 @@ uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" version = "1.4.3" [[deps.Statistics]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.11.1" -weakdeps = ["SparseArrays"] - - [deps.Statistics.extensions] - SparseArraysExt = ["SparseArrays"] +version = "1.10.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] @@ -835,7 +810,7 @@ weakdeps = ["Adapt", "GPUArraysCore", "SparseArrays", "StaticArrays"] [[deps.SuiteSparse_jll]] deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.7.0+0" +version = "7.2.1+1" [[deps.TOML]] deps = ["Dates"] @@ -876,7 +851,6 @@ version = "0.17.3" [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -version = "1.11.0" [[deps.TimerOutputs]] deps = ["ExprTools", "Printf"] @@ -892,11 +866,9 @@ version = "0.11.3" [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -version = "1.11.0" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" [[deps.UnsafeAtomics]] git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" @@ -958,7 +930,7 @@ version = "1.11.1+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.59.0+0" +version = "1.52.0+1" [[deps.oneTBB_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] From 59548f880a81a59c172f20dbcc61a737d3cd24e0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 10:06:01 +0100 Subject: [PATCH 21/48] we probably need to initialize on a GPU --- .buildkite/distributed/pipeline.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index e0ea62eef4..6c82558a3e 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -5,13 +5,15 @@ agents: env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" - JULIA_MPI_HAS_CUDA: "true" OPENBLAS_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: - label: "initialize" key: "init_central" + env: + TEST_GROUP: "init" + GPU_TEST: "true" command: - echo "--- Instantiate project" - "julia --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" @@ -25,6 +27,10 @@ steps: - echo "--- Instantiate status" - "julia --project -e 'using Pkg; Pkg.status()'" + agents: + slurm_mem: 120G + slurm_ntasks: 1 + slurm_gpus_per_task: 1 - wait From 642cfd9d594dc7503e4e5ad187cda800c87a5b26 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 10:06:43 +0100 Subject: [PATCH 22/48] these options should not create problems --- .buildkite/distributed/pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 6c82558a3e..6b993bbec2 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -6,6 +6,9 @@ agents: env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 + JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager + JULIA_NUM_PRECOMPILE_TASKS: 8 + JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: From 4cee49ab8be4408fbba12cd23f0c826c48b927d1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 17:18:36 +0100 Subject: [PATCH 23/48] let's see if this differs --- test/runtests.jl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 11a4e553e8..f4f0d532ca 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -179,7 +179,18 @@ CUDA.allowscalar() do if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.6") + Pkg.instantiate(; verbose=true) + Pkg.precompile(; strict=true) + Pkg.status() + + try + MPI.versioninfo() + catch; end + + try + CUDA.precompile_runtime() + CUDA.versioninfo() + catch; end include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end From a46b25dee593075c6a60e0f58728cd6d7866accc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 18:57:36 +0100 Subject: [PATCH 24/48] just version infos --- test/runtests.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f4f0d532ca..69de014ec9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,8 +28,6 @@ CUDA.allowscalar() do # Initialization steps if group == :init || group == :all - Pkg.instantiate(; verbose=true) - Pkg.precompile(; strict=true) Pkg.status() try @@ -37,7 +35,6 @@ CUDA.allowscalar() do catch; end try - CUDA.precompile_runtime() CUDA.versioninfo() catch; end end From 4dffbe5d40cf36bb54c974ef3cf51da9013315c0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 09:28:16 +0100 Subject: [PATCH 25/48] fiddling with O0 --- .buildkite/distributed/pipeline.yml | 6 +- Manifest.toml | 944 ---------------------------- 2 files changed, 3 insertions(+), 947 deletions(-) delete mode 100644 Manifest.toml diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 6b993bbec2..7c892803a1 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -19,11 +19,11 @@ steps: GPU_TEST: "true" command: - echo "--- Instantiate project" - - "julia --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" + - "julia -O0 --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" # Force the initialization of the CUDA runtime as it is lazily loaded by default - - "julia --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - - "julia --project -e 'using MPI; MPI.versioninfo()'" + - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" + - "julia -O0 --project -e 'using MPI; MPI.versioninfo()'" # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" diff --git a/Manifest.toml b/Manifest.toml deleted file mode 100644 index 45d969c32d..0000000000 --- a/Manifest.toml +++ /dev/null @@ -1,944 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.10.6" -manifest_format = "2.0" -project_hash = "475d8d28ec1bf53471638e448deed724aee15ccd" - -[[deps.AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "1.5.0" - - [deps.AbstractFFTs.extensions] - AbstractFFTsChainRulesCoreExt = "ChainRulesCore" - AbstractFFTsTestExt = "Test" - - [deps.AbstractFFTs.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.Adapt]] -deps = ["LinearAlgebra", "Requires"] -git-tree-sha1 = "50c3c56a52972d78e8be9fd135bfb91c9574c140" -uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "4.1.1" -weakdeps = ["StaticArrays"] - - [deps.Adapt.extensions] - AdaptStaticArraysExt = "StaticArrays" - -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.1" - -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[deps.Atomix]] -deps = ["UnsafeAtomics"] -git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" -uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" -version = "0.1.0" - -[[deps.BFloat16s]] -deps = ["LinearAlgebra", "Printf", "Random", "Test"] -git-tree-sha1 = "2c7cc21e8678eff479978a0a2ef5ce2f51b63dff" -uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" -version = "0.5.0" - -[[deps.Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[deps.Blosc_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] -git-tree-sha1 = "ef12cdd1c7fb7e1dfd6fa8fd60d4db6bc61d2f23" -uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9" -version = "1.21.6+0" - -[[deps.Bzip2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "8873e196c2eb87962a2048b3b8e08946535864a1" -uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" -version = "1.0.8+2" - -[[deps.CEnum]] -git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" -uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" -version = "0.5.0" - -[[deps.CFTime]] -deps = ["Dates", "Printf"] -git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1" -uuid = "179af706-886a-5703-950a-314cd64e0468" -version = "0.1.3" - -[[deps.CUDA]] -deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "StaticArrays", "Statistics", "demumble_jll"] -git-tree-sha1 = "e0725a467822697171af4dae15cec10b4fc19053" -uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "5.5.2" - - [deps.CUDA.extensions] - ChainRulesCoreExt = "ChainRulesCore" - EnzymeCoreExt = "EnzymeCore" - SpecialFunctionsExt = "SpecialFunctions" - - [deps.CUDA.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" - SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" - -[[deps.CUDA_Driver_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "ccd1e54610c222fadfd4737dac66bff786f63656" -uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" -version = "0.10.3+0" - -[[deps.CUDA_Runtime_Discovery]] -deps = ["Libdl"] -git-tree-sha1 = "33576c7c1b2500f8e7e6baa082e04563203b3a45" -uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" -version = "0.3.5" - -[[deps.CUDA_Runtime_jll]] -deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "e43727b237b2879a34391eeb81887699a26f8f2f" -uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -version = "0.15.3+0" - -[[deps.ColorTypes]] -deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" -uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.11.5" - -[[deps.Colors]] -deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] -git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" -uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.12.11" - -[[deps.CommonDataModel]] -deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] -git-tree-sha1 = "d6fb5bf939a2753c74984b11434ea25d6c397a58" -uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" -version = "0.3.6" - -[[deps.Compat]] -deps = ["TOML", "UUIDs"] -git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.16.0" -weakdeps = ["Dates", "LinearAlgebra"] - - [deps.Compat.extensions] - CompatLinearAlgebraExt = "LinearAlgebra" - -[[deps.CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.1.1+0" - -[[deps.ConstructionBase]] -git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" -uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.5.8" - - [deps.ConstructionBase.extensions] - ConstructionBaseIntervalSetsExt = "IntervalSets" - ConstructionBaseLinearAlgebraExt = "LinearAlgebra" - ConstructionBaseStaticArraysExt = "StaticArrays" - - [deps.ConstructionBase.weakdeps] - IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" - LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - -[[deps.Crayons]] -git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.1.1" - -[[deps.CubedSphere]] -deps = ["TaylorSeries"] -git-tree-sha1 = "51bb25de518b4c62b7cdf26e5fbb84601bb27a60" -uuid = "7445602f-e544-4518-8976-18f8e8ae6cdb" -version = "0.3.0" - -[[deps.DataAPI]] -git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.16.0" - -[[deps.DataFrames]] -deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6" -uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.7.0" - -[[deps.DataStructures]] -deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.20" - -[[deps.DataValueInterfaces]] -git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" -uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" -version = "1.0.0" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[deps.DiskArrays]] -deps = ["LRUCache", "OffsetArrays"] -git-tree-sha1 = "e0e89a60637a62d13aa2107f0acd169b9b9b77e7" -uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" -version = "0.4.6" - -[[deps.Distances]] -deps = ["LinearAlgebra", "Statistics", "StatsAPI"] -git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" -uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" -version = "0.10.12" - - [deps.Distances.extensions] - DistancesChainRulesCoreExt = "ChainRulesCore" - DistancesSparseArraysExt = "SparseArrays" - - [deps.Distances.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[deps.Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" - -[[deps.ExprTools]] -git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" -uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.10" - -[[deps.FFTW]] -deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] -git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" -uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.8.0" - -[[deps.FFTW_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4d81ed14783ec49ce9f2e168208a12ce1815aa25" -uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" -version = "3.3.10+1" - -[[deps.FileIO]] -deps = ["Pkg", "Requires", "UUIDs"] -git-tree-sha1 = "62ca0547a14c57e98154423419d8a342dca75ca9" -uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -version = "1.16.4" - -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" - -[[deps.FixedPointNumbers]] -deps = ["Statistics"] -git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" -uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -version = "0.8.5" - -[[deps.Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - -[[deps.GMP_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" -version = "6.2.1+6" - -[[deps.GPUArrays]] -deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] -git-tree-sha1 = "62ee71528cca49be797076a76bdc654a170a523e" -uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "10.3.1" - -[[deps.GPUArraysCore]] -deps = ["Adapt"] -git-tree-sha1 = "ec632f177c0d990e64d955ccc1b8c04c485a0950" -uuid = "46192b85-c4d5-4398-a991-12ede77f4527" -version = "0.1.6" - -[[deps.GPUCompiler]] -deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "PrecompileTools", "Preferences", "Scratch", "Serialization", "TOML", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "1d6f290a5eb1201cd63574fbc4440c788d5cb38f" -uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.27.8" - -[[deps.Glob]] -git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" -uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" -version = "1.3.1" - -[[deps.GnuTLS_jll]] -deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Nettle_jll", "P11Kit_jll", "Zlib_jll"] -git-tree-sha1 = "383db7d3f900f4c1f47a8a04115b053c095e48d3" -uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" -version = "3.8.4+0" - -[[deps.HDF5_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] -git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" -uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" -version = "1.14.2+1" - -[[deps.Hwloc_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "dd3b49277ec2bb2c6b94eb1604d4d0616016f7a6" -uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" -version = "2.11.2+0" - -[[deps.IncompleteLU]] -deps = ["LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "6c676e79f98abb6d33fa28122cad099f1e464afe" -uuid = "40713840-3770-5561-ab4c-a76e7d0d7895" -version = "0.2.1" - -[[deps.InlineStrings]] -git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" -uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.4.2" - - [deps.InlineStrings.extensions] - ArrowTypesExt = "ArrowTypes" - ParsersExt = "Parsers" - - [deps.InlineStrings.weakdeps] - ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" - Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" - -[[deps.IntelOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] -git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e" -uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" -version = "2024.2.1+0" - -[[deps.InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[deps.InvertedIndices]] -git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" -uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.3.0" - -[[deps.IterativeSolvers]] -deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] -git-tree-sha1 = "59545b0a2b27208b0650df0a46b8e3019f85055b" -uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" -version = "0.9.4" - -[[deps.IteratorInterfaceExtensions]] -git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" -uuid = "82899510-4779-5014-852e-03e436cf321d" -version = "1.0.0" - -[[deps.JLD2]] -deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] -git-tree-sha1 = "783c1be5213a09609b23237a0c9e5dfd258ae6f2" -uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.5.7" - -[[deps.JLLWrappers]] -deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.6.1" - -[[deps.JuliaNVTXCallbacks_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "af433a10f3942e882d3c671aacb203e006a5808f" -uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e" -version = "0.2.1+0" - -[[deps.KernelAbstractions]] -deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "e73a077abc7fe798fe940deabe30ef6c66bdde52" -uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" -version = "0.9.29" - - [deps.KernelAbstractions.extensions] - EnzymeExt = "EnzymeCore" - LinearAlgebraExt = "LinearAlgebra" - SparseArraysExt = "SparseArrays" - - [deps.KernelAbstractions.weakdeps] - EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" - LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[deps.LLVM]] -deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Unicode"] -git-tree-sha1 = "d422dfd9707bec6617335dc2ea3c5172a87d5908" -uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "9.1.3" -weakdeps = ["BFloat16s"] - - [deps.LLVM.extensions] - BFloat16sExt = "BFloat16s" - -[[deps.LLVMExtra_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "05a8bd5a42309a9ec82f700876903abce1017dd3" -uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" -version = "0.0.34+0" - -[[deps.LLVMLoopInfo]] -git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" -uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" -version = "1.0.0" - -[[deps.LLVMOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" -uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" -version = "18.1.7+0" - -[[deps.LRUCache]] -git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" -uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" -version = "1.6.1" -weakdeps = ["Serialization"] - - [deps.LRUCache.extensions] - SerializationExt = ["Serialization"] - -[[deps.LaTeXStrings]] -git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" -uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" -version = "1.4.0" - -[[deps.LazyArtifacts]] -deps = ["Artifacts", "Pkg"] -uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" - -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.4" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.4.0+0" - -[[deps.LibGit2]] -deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.6.4+0" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.0+1" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[deps.Libiconv_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" -uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.17.0+1" - -[[deps.LinearAlgebra]] -deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[deps.Lz4_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "abf88ff67f4fd89839efcae2f4c39cbc4ecd0846" -uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" -version = "1.10.0+1" - -[[deps.MKL_jll]] -deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] -git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f" -uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" -version = "2024.2.0+0" - -[[deps.MPI]] -deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "PkgVersion", "PrecompileTools", "Requires", "Serialization", "Sockets"] -git-tree-sha1 = "892676019c58f34e38743bc989b0eca5bce5edc5" -uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195" -version = "0.20.22" - - [deps.MPI.extensions] - AMDGPUExt = "AMDGPU" - CUDAExt = "CUDA" - - [deps.MPI.weakdeps] - AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" - CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" - -[[deps.MPICH_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] -git-tree-sha1 = "7715e65c47ba3941c502bffb7f266a41a7f54423" -uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" -version = "4.2.3+0" - -[[deps.MPIPreferences]] -deps = ["Libdl", "Preferences"] -git-tree-sha1 = "c105fe467859e7f6e9a852cb15cb4301126fac07" -uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" -version = "0.1.11" - -[[deps.MPItrampoline_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] -git-tree-sha1 = "70e830dab5d0775183c99fc75e4c24c614ed7142" -uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" -version = "5.5.1+0" - -[[deps.MacroTools]] -deps = ["Markdown", "Random"] -git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" -uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.13" - -[[deps.Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[deps.MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+1" - -[[deps.MicrosoftMPI_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" -uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" -version = "10.1.4+2" - -[[deps.Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.2.0" - -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[deps.MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.1.10" - -[[deps.NCDatasets]] -deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] -git-tree-sha1 = "2c9dc92001ac06d432f363f37ff5552954d9947c" -uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" -version = "0.14.6" - -[[deps.NVTX]] -deps = ["Colors", "JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"] -git-tree-sha1 = "53046f0483375e3ed78e49190f1154fa0a4083a1" -uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f" -version = "0.3.4" - -[[deps.NVTX_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "ce3269ed42816bf18d500c9f63418d4b0d9f5a3b" -uuid = "e98f9f5b-d649-5603-91fd-7774390e6439" -version = "3.1.0+2" - -[[deps.NetCDF_jll]] -deps = ["Artifacts", "Blosc_jll", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenMPI_jll", "XML2_jll", "Zlib_jll", "Zstd_jll", "libzip_jll"] -git-tree-sha1 = "a8af1798e4eb9ff768ce7fdefc0e957097793f15" -uuid = "7243133f-43d8-5620-bbf4-c2c921802cf3" -version = "400.902.209+0" - -[[deps.Nettle_jll]] -deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "eca63e3847dad608cfa6a3329b95ef674c7160b4" -uuid = "4c82536e-c426-54e4-b420-14f461c4ed8b" -version = "3.7.2+0" - -[[deps.NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" - -[[deps.OffsetArrays]] -git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e" -uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "1.14.1" -weakdeps = ["Adapt"] - - [deps.OffsetArrays.extensions] - OffsetArraysAdaptExt = "Adapt" - -[[deps.OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.23+4" - -[[deps.OpenMPI_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] -git-tree-sha1 = "bfce6d523861a6c562721b262c0d1aaeead2647f" -uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" -version = "5.0.5+0" - -[[deps.OpenSSL_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" -uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+1" - -[[deps.OrderedCollections]] -git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.6.3" - -[[deps.P11Kit_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" -uuid = "c2071276-7c44-58a7-b746-946036e04d0a" -version = "0.24.1+0" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.10.0" - -[[deps.PkgVersion]] -deps = ["Pkg"] -git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" -uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" -version = "0.3.3" - -[[deps.PooledArrays]] -deps = ["DataAPI", "Future"] -git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" -uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" -version = "1.4.3" - -[[deps.PrecompileTools]] -deps = ["Preferences"] -git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" -uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.1" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" - -[[deps.PrettyTables]] -deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34" -uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.4.0" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[deps.Quaternions]] -deps = ["LinearAlgebra", "Random", "RealDot"] -git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" -uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" -version = "0.7.6" - -[[deps.REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[deps.Random]] -deps = ["SHA"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[deps.Random123]] -deps = ["Random", "RandomNumbers"] -git-tree-sha1 = "4743b43e5a9c4a2ede372de7061eed81795b12e7" -uuid = "74087812-796a-5b5d-8853-05524746bad3" -version = "1.7.0" - -[[deps.RandomNumbers]] -deps = ["Random"] -git-tree-sha1 = "c6ec94d2aaba1ab2ff983052cf6a606ca5985902" -uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" -version = "1.6.0" - -[[deps.RealDot]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" -uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" -version = "0.1.0" - -[[deps.RecipesBase]] -deps = ["PrecompileTools"] -git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" -uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" -version = "1.3.4" - -[[deps.Reexport]] -git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "1.2.2" - -[[deps.Requires]] -deps = ["UUIDs"] -git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" -uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "1.3.0" - -[[deps.Rotations]] -deps = ["LinearAlgebra", "Quaternions", "Random", "StaticArrays"] -git-tree-sha1 = "5680a9276685d392c87407df00d57c9924d9f11e" -uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" -version = "1.7.1" -weakdeps = ["RecipesBase"] - - [deps.Rotations.extensions] - RotationsRecipesBaseExt = "RecipesBase" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.Scratch]] -deps = ["Dates"] -git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" -uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.2.1" - -[[deps.SeawaterPolynomials]] -git-tree-sha1 = "78f965a2f0cd5250a20c9aba9979346dd2b35734" -uuid = "d496a93d-167e-4197-9f49-d3af4ff8fe40" -version = "0.3.5" - -[[deps.SentinelArrays]] -deps = ["Dates", "Random"] -git-tree-sha1 = "d0553ce4031a081cc42387a9b9c8441b7d99f32d" -uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.4.7" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[deps.Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[deps.SortingAlgorithms]] -deps = ["DataStructures"] -git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.2.1" - -[[deps.SparseArrays]] -deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.10.0" - -[[deps.StaticArrays]] -deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] -git-tree-sha1 = "777657803913ffc7e8cc20f0fd04b634f871af8f" -uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.9.8" - - [deps.StaticArrays.extensions] - StaticArraysChainRulesCoreExt = "ChainRulesCore" - StaticArraysStatisticsExt = "Statistics" - - [deps.StaticArrays.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - -[[deps.StaticArraysCore]] -git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" -uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" -version = "1.4.3" - -[[deps.Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.10.0" - -[[deps.StatsAPI]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" -uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.7.0" - -[[deps.StringManipulation]] -deps = ["PrecompileTools"] -git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" -uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.4.0" - -[[deps.StructArrays]] -deps = ["ConstructionBase", "DataAPI", "Tables"] -git-tree-sha1 = "f4dc295e983502292c4c3f951dbb4e985e35b3be" -uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" -version = "0.6.18" -weakdeps = ["Adapt", "GPUArraysCore", "SparseArrays", "StaticArrays"] - - [deps.StructArrays.extensions] - StructArraysAdaptExt = "Adapt" - StructArraysGPUArraysCoreExt = "GPUArraysCore" - StructArraysSparseArraysExt = "SparseArrays" - StructArraysStaticArraysExt = "StaticArrays" - -[[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] -uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.2.1+1" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.TableTraits]] -deps = ["IteratorInterfaceExtensions"] -git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" -uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" -version = "1.0.1" - -[[deps.Tables]] -deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] -git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" -uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.12.0" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" - -[[deps.TaylorSeries]] -deps = ["LinearAlgebra", "Markdown", "Requires", "SparseArrays"] -git-tree-sha1 = "abc13c4d3cccd1703335ba03abdaf0dc8ecc97e2" -uuid = "6aa5eb33-94cf-58f4-a9d0-e4b2c4fc25ea" -version = "0.17.3" - - [deps.TaylorSeries.extensions] - TaylorSeriesIAExt = "IntervalArithmetic" - TaylorSeriesSAExt = "StaticArrays" - - [deps.TaylorSeries.weakdeps] - IntervalArithmetic = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - -[[deps.Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.TimerOutputs]] -deps = ["ExprTools", "Printf"] -git-tree-sha1 = "3a6f063d690135f5c1ba351412c82bae4d1402bf" -uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.25" - -[[deps.TranscodingStreams]] -git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" -uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.11.3" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[deps.UnsafeAtomics]] -git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" -uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" -version = "0.2.1" - -[[deps.UnsafeAtomicsLLVM]] -deps = ["LLVM", "UnsafeAtomics"] -git-tree-sha1 = "2d17fabcd17e67d7625ce9c531fb9f40b7c42ce4" -uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" -version = "0.2.1" - -[[deps.XML2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] -git-tree-sha1 = "6a451c6f33a176150f315726eba8b92fbfdb9ae7" -uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.13.4+0" - -[[deps.XZ_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "15e637a697345f6743674f1322beefbc5dcd5cfc" -uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" -version = "5.6.3+0" - -[[deps.Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+1" - -[[deps.Zstd_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" -uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" -version = "1.5.6+1" - -[[deps.demumble_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6498e3581023f8e530f34760d18f75a69e3a4ea8" -uuid = "1e29f10c-031c-5a83-9565-69cddfc27673" -version = "1.3.0+0" - -[[deps.libaec_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" -uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" -version = "1.1.2+0" - -[[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.11.0+0" - -[[deps.libzip_jll]] -deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] -git-tree-sha1 = "668ac0297e6bd8f4d53dfdcd3ace71f2e00f4a35" -uuid = "337d8026-41b4-5cde-a456-74a10e5b31d1" -version = "1.11.1+0" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.52.0+1" - -[[deps.oneTBB_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "7d0ea0f4895ef2f5cb83645fa689e52cb55cf493" -uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" -version = "2021.12.0+0" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+2" From 9c3c6cd75c4f5892cbf8d5be61a5b78ad0754c9b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 10:56:50 +0100 Subject: [PATCH 26/48] why are we using 8 threads? --- .buildkite/distributed/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 7c892803a1..874f939bb1 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -7,8 +7,8 @@ env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 8 + JULIA_NUM_PRECOMPILE_TASKS: 1 + JULIA_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: From 3b28ecb2c4d7cdf9be60def791e3f2d03f79e444 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 11:03:56 +0100 Subject: [PATCH 27/48] memory requirements are not this huge --- .buildkite/distributed/pipeline.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 874f939bb1..20d2eed28a 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -31,7 +31,7 @@ steps: - echo "--- Instantiate status" - "julia --project -e 'using Pkg; Pkg.status()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 1 slurm_gpus_per_task: 1 @@ -45,7 +45,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🐲 gpu distributed unit tests" @@ -57,7 +57,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 @@ -69,7 +69,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🛸 gpu distributed solvers tests" @@ -81,7 +81,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -97,7 +97,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🦏 gpu distributed hydrostatic model tests" @@ -109,7 +109,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -125,7 +125,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🕺 gpu distributed nonhydrostatic regression" @@ -137,7 +137,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: From 7126c7cb5c99c464625b6c970733bc0dc4478254 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 11:49:08 +0100 Subject: [PATCH 28/48] speed up the precompilation a bit, to revert later --- .buildkite/distributed/pipeline.yml | 6 ------ Project.toml | 3 +-- test/dependencies_for_runtests.jl | 2 +- test/runtests.jl | 12 ++++++------ 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 20d2eed28a..ce9f53d027 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -24,12 +24,6 @@ steps: # Force the initialization of the CUDA runtime as it is lazily loaded by default - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "julia -O0 --project -e 'using MPI; MPI.versioninfo()'" - - # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl - - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" - - - echo "--- Instantiate status" - - "julia --project -e 'using Pkg; Pkg.status()'" agents: slurm_mem: 8G slurm_ntasks: 1 diff --git a/Project.toml b/Project.toml index 4d80cee5b7..33fa9f8b01 100644 --- a/Project.toml +++ b/Project.toml @@ -77,10 +77,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" -Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "SafeTestsets", "Test", "TimesDates"] diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 3e15e0a3d8..54da7bbc52 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -using Enzyme +# using Enzyme using SparseArrays using JLD2 using FFTW diff --git a/test/runtests.jl b/test/runtests.jl index 69de014ec9..28bf877c2a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -221,12 +221,12 @@ CUDA.allowscalar() do end end - # Tests for Enzyme extension - if group == :enzyme || group == :all - @testset "Enzyme extension tests" begin - include("test_enzyme.jl") - end - end + # # Tests for Enzyme extension + # if group == :enzyme || group == :all + # @testset "Enzyme extension tests" begin + # include("test_enzyme.jl") + # end + # end if group == :convergence include("test_convergence.jl") From 733ab2bc614cd338a96bfc37f620f5e118d4d17e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:02:32 +0100 Subject: [PATCH 29/48] might this be the culprit? --- .buildkite/distributed/pipeline.yml | 3 +++ Project.toml | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index ce9f53d027..f13d8ae8cb 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -24,6 +24,9 @@ steps: # Force the initialization of the CUDA runtime as it is lazily loaded by default - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "julia -O0 --project -e 'using MPI; MPI.versioninfo()'" + + - echo "--- Initialize tests" + - "julia -O0 --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 8G slurm_ntasks: 1 diff --git a/Project.toml b/Project.toml index 33fa9f8b01..a0b9ffd19f 100644 --- a/Project.toml +++ b/Project.toml @@ -80,6 +80,7 @@ DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" +MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" [targets] -test = ["DataDeps", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "SafeTestsets", "Test", "MPIPreferences", "TimesDates"] From 2dbf1a0c5da4338257f7b90c715b4743ba6fcfb6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:03:25 +0100 Subject: [PATCH 30/48] revert to 8 tasks to precompile --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index f13d8ae8cb..1eb73a04ec 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -7,7 +7,7 @@ env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 1 + JULIA_NUM_PRECOMPILE_TASKS: 8 JULIA_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 From a4b129ade57a5cc4d1efbdc8460d6b5cbd0e4655 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:04:19 +0100 Subject: [PATCH 31/48] final version? --- .buildkite/distributed/pipeline.yml | 20 ++++++++++++++++++++ test/runtests.jl | 3 --- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 1eb73a04ec..fa620cd04d 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -44,6 +44,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🐲 gpu distributed unit tests" key: "distributed_gpu" @@ -57,6 +61,10 @@ steps: slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🦾 cpu distributed solvers tests" key: "distributed_solvers_cpu" @@ -68,6 +76,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🛸 gpu distributed solvers tests" key: "distributed_solvers_gpu" @@ -96,6 +108,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🦏 gpu distributed hydrostatic model tests" key: "distributed_hydrostatic_model_gpu" @@ -124,6 +140,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🕺 gpu distributed nonhydrostatic regression" key: "distributed_nonhydrostatic_regression_gpu" diff --git a/test/runtests.jl b/test/runtests.jl index 28bf877c2a..9661096837 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -170,7 +170,6 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() - CUDA.set_runtime_version!(v"12.6") include("test_distributed_models.jl") end @@ -195,14 +194,12 @@ CUDA.allowscalar() do if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() - CUDA.set_runtime_version!(v"12.6") include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") end if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.6") archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From 29f7d69a7d2814aec9f7acabefc3191001aac4f3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:28:55 +0100 Subject: [PATCH 32/48] return to previous state of affairs --- Project.toml | 5 +++-- test/runtests.jl | 24 ++++++------------------ test/utils_for_runtests.jl | 4 ++-- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/Project.toml b/Project.toml index a0b9ffd19f..c12724ce34 100644 --- a/Project.toml +++ b/Project.toml @@ -50,7 +50,7 @@ CubedSphere = "0.2, 0.3" Dates = "1.9" Distances = "0.10" DocStringExtensions = "0.8, 0.9" -Enzyme = "0.13.3" +Enzyme = "0.13.14" FFTW = "1" Glob = "1.3" IncompleteLU = "0.2" @@ -80,7 +80,8 @@ DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" [targets] -test = ["DataDeps", "SafeTestsets", "Test", "MPIPreferences", "TimesDates"] +test = ["DataDeps", "SafeTestsets", "Test", "Enzyme", "MPIPreferences", "TimesDates"] diff --git a/test/runtests.jl b/test/runtests.jl index 9661096837..bbf6415976 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -175,18 +175,6 @@ CUDA.allowscalar() do if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() - Pkg.instantiate(; verbose=true) - Pkg.precompile(; strict=true) - Pkg.status() - - try - MPI.versioninfo() - catch; end - - try - CUDA.precompile_runtime() - CUDA.versioninfo() - catch; end include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end @@ -218,12 +206,12 @@ CUDA.allowscalar() do end end - # # Tests for Enzyme extension - # if group == :enzyme || group == :all - # @testset "Enzyme extension tests" begin - # include("test_enzyme.jl") - # end - # end + # Tests for Enzyme extension + if group == :enzyme || group == :all + @testset "Enzyme extension tests" begin + include("test_enzyme.jl") + end + end if group == :convergence include("test_convergence.jl") diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index a64d44263b..f27ba77054 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -5,8 +5,8 @@ import Oceananigans.Fields: interior # Are the test running on the GPUs? # Are the test running in parallel? -child_arch = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() -mpi_test = parse(Bool, get(ENV, "MPI_TEST", "false")) +child_arch = get(ENV, "GPU_TEST", nothing) == "true" ? GPU() : CPU() +mpi_test = get(ENV, "MPI_TEST", nothing) == "true" function test_architectures() # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. From b1743134804f6ffce3781ae72b96d8edfcfea845 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:32:13 +0100 Subject: [PATCH 33/48] reinclude enzyme --- test/dependencies_for_runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 54da7bbc52..3e15e0a3d8 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -# using Enzyme +using Enzyme using SparseArrays using JLD2 using FFTW From 0283e6a5ac27673ad3f7d18eca74e0b616428a0f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 14:57:05 +0100 Subject: [PATCH 34/48] set cuda runtime version --- test/runtests.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index bbf6415976..8a1cff05f3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -169,18 +169,21 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = test_architectures() include("test_distributed_models.jl") end if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = test_architectures() include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") @@ -188,6 +191,7 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From b4c1f2a6f930987cd0ede749ddcf29471ccb7800 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 16:03:20 +0100 Subject: [PATCH 35/48] will this help in finding cuda? --- .buildkite/distributed/pipeline.yml | 4 ++++ test/runtests.jl | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index fa620cd04d..7e485690cb 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -56,6 +56,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 8G @@ -88,6 +89,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 8G @@ -120,6 +122,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 8G @@ -152,6 +155,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 8G diff --git a/test/runtests.jl b/test/runtests.jl index 8a1cff05f3..bbf6415976 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -169,21 +169,18 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = test_architectures() include("test_distributed_models.jl") end if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = test_architectures() include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") @@ -191,7 +188,6 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From bc53a977c21f5063b95d251ca3c8a4abeeeaf80a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 17:07:54 +0100 Subject: [PATCH 36/48] make sure we don't run OOM --- .buildkite/distributed/pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 7e485690cb..f76f2fced8 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -8,7 +8,7 @@ env: OPENBLAS_NUM_THREADS: 1 JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 1 + JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: @@ -92,7 +92,7 @@ steps: - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 8G + slurm_mem: 32G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -108,7 +108,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 8G + slurm_mem: 32G slurm_ntasks: 4 retry: automatic: @@ -125,7 +125,7 @@ steps: - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 8G + slurm_mem: 32G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -141,7 +141,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 8G + slurm_mem: 32G slurm_ntasks: 4 retry: automatic: @@ -158,7 +158,7 @@ steps: - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 8G + slurm_mem: 32G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: From 811bfdbdcd77f53ab6302ff1b132e74ee8adddd7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 09:13:04 +0100 Subject: [PATCH 37/48] bugfix in `set!` --- src/Fields/set!.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Fields/set!.jl b/src/Fields/set!.jl index e4b8a29cb6..a81e8c273b 100644 --- a/src/Fields/set!.jl +++ b/src/Fields/set!.jl @@ -3,7 +3,7 @@ using KernelAbstractions: @kernel, @index using Adapt: adapt_structure using Oceananigans.Grids: on_architecture, node_names -using Oceananigans.Architectures: child_architecture, device, GPU, CPU +using Oceananigans.Architectures: child_architecture, cpu_architecture, device, GPU, CPU using Oceananigans.Utils: work_layout ##### @@ -49,8 +49,10 @@ function set_to_function!(u, f) # Determine cpu_grid and cpu_u if arch isa GPU - cpu_grid = on_architecture(CPU(), u.grid) - cpu_u = Field(location(u), cpu_grid; indices = indices(u)) + cpu_arch = cpu_architecture(arch) + cpu_grid = on_architecture(cpu_arch, u.grid) + cpu_u = Field(location(u), cpu_grid; indices = indices(u)) + elseif arch isa CPU cpu_grid = u.grid cpu_u = u @@ -65,8 +67,8 @@ function set_to_function!(u, f) catch err u_loc = Tuple(L() for L in location(u)) - arg_str = tuple_string(node_names(u.grid, u_loc...)) - loc_str = tuple_string(location(u)) + arg_str = tuple_string(node_names(u.grid, u_loc...)) + loc_str = tuple_string(location(u)) topo_str = tuple_string(topology(u.grid)) msg = string("An error was encountered within set! while setting the field", '\n', '\n', @@ -81,10 +83,8 @@ function set_to_function!(u, f) end # Transfer data to GPU if u is on the GPU - if child_architecture(u) isa GPU - set!(u, cpu_u) - end - + arch isa GPU && set!(u, cpu_u) + return u end From cd86a6ac4b55337057e1056dbc5a384ef355d0b5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 09:25:13 +0100 Subject: [PATCH 38/48] try precompile inside runtests --- .buildkite/distributed/pipeline.yml | 4 ---- test/runtests.jl | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index f76f2fced8..95ce6a3123 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -56,7 +56,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 8G @@ -89,7 +88,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 32G @@ -122,7 +120,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 32G @@ -155,7 +152,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 32G diff --git a/test/runtests.jl b/test/runtests.jl index bbf6415976..07bc2bf495 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -169,18 +169,21 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() + CUDA.precompile_runtime() archs = test_architectures() include("test_distributed_models.jl") end if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() + CUDA.precompile_runtime() include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() + CUDA.precompile_runtime() archs = test_architectures() include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") @@ -188,6 +191,7 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() + CUDA.precompile_runtime() archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From 4039299db4b4bf14575929062a689ef73b4c7de1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 09:59:37 +0100 Subject: [PATCH 39/48] revert back --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 07bc2bf495..36b146605c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,6 +28,8 @@ CUDA.allowscalar() do # Initialization steps if group == :init || group == :all + Pkg.instantiate(; verbose=true) + Pkg.precompile(; strict=true) Pkg.status() try @@ -35,6 +37,7 @@ CUDA.allowscalar() do catch; end try + CUDA.precompile_runtime() CUDA.versioninfo() catch; end end @@ -176,14 +179,12 @@ CUDA.allowscalar() do if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() - CUDA.precompile_runtime() include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() - CUDA.precompile_runtime() archs = test_architectures() include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") @@ -191,7 +192,6 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() - CUDA.precompile_runtime() archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From 2c6ad90b5d19953502b95c98388d4abaf8f840ff Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 10:37:37 +0100 Subject: [PATCH 40/48] recompile everywhere --- test/runtests.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 36b146605c..1049035e14 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -179,12 +179,14 @@ CUDA.allowscalar() do if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() + CUDA.precompile_runtime() include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() + CUDA.precompile_runtime() archs = test_architectures() include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") @@ -192,6 +194,7 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() + CUDA.precompile_runtime() archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From 781992c65198d5327e37236c4c6dda02cbbb1632 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 11:42:36 +0100 Subject: [PATCH 41/48] try nuclear option --- Project.toml | 7 ++++--- test/runtests.jl | 8 ++++---- test/utils_for_runtests.jl | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index c12724ce34..9b7c7540d1 100644 --- a/Project.toml +++ b/Project.toml @@ -77,11 +77,12 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" -Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" [targets] -test = ["DataDeps", "SafeTestsets", "Test", "Enzyme", "MPIPreferences", "TimesDates"] +test = ["DataDeps", "SafeTestsets", "Test", "Enzyme", "CUDA_Runtime_jll", "MPIPreferences", "TimesDates"] diff --git a/test/runtests.jl b/test/runtests.jl index 1049035e14..9c07f19b9a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -172,21 +172,21 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() - CUDA.precompile_runtime() + reset_cuda_if_necessary() archs = test_architectures() include("test_distributed_models.jl") end if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() - CUDA.precompile_runtime() + reset_cuda_if_necessary() include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() - CUDA.precompile_runtime() + reset_cuda_if_necessary() archs = test_architectures() include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") @@ -194,7 +194,7 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() - CUDA.precompile_runtime() + reset_cuda_if_necessary() archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index f27ba77054..8dcf7fbf58 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -8,6 +8,27 @@ import Oceananigans.Fields: interior child_arch = get(ENV, "GPU_TEST", nothing) == "true" ? GPU() : CPU() mpi_test = get(ENV, "MPI_TEST", nothing) == "true" +function reset_cuda_if_necessary() + + # Do nothing if we are on the CPU + if child_arch isa CPU + return + end + + try + c = CUDA.zeros(10) # This will fail if CUDA is not available + catch err + if MPI.Comm_rank(MPI.COMM_WORLD) == 0 + pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll") + Base.compilecache(pkg) + @info "CUDA.jl was not loaded. Re-loading CUDA.jl and re-starting Julia." + end + MPI.Barrier(MPI.COMM_WORLD) + # re-start Julia and re-load CUDA.jl + throw(err) + end +end + function test_architectures() # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test several different configurations: `Partition(x = 4)`, `Partition(y = 4)`, From 08949b3214aa7870987611b7996700491ef42147 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 11:45:08 +0100 Subject: [PATCH 42/48] skip all these commands --- .buildkite/distributed/pipeline.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 95ce6a3123..c4e1d0cc57 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -18,13 +18,6 @@ steps: TEST_GROUP: "init" GPU_TEST: "true" command: - - echo "--- Instantiate project" - - "julia -O0 --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" - - # Force the initialization of the CUDA runtime as it is lazily loaded by default - - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - - "julia -O0 --project -e 'using MPI; MPI.versioninfo()'" - - echo "--- Initialize tests" - "julia -O0 --project -e 'using Pkg; Pkg.test()'" agents: From 908b31a67d1e8d4011d790746777fee3b63b170a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 12:46:14 +0100 Subject: [PATCH 43/48] some failsafe option --- src/Fields/set!.jl | 9 +++++---- test/runtests.jl | 4 ++++ test/utils_for_runtests.jl | 10 +++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Fields/set!.jl b/src/Fields/set!.jl index a81e8c273b..e311f659f1 100644 --- a/src/Fields/set!.jl +++ b/src/Fields/set!.jl @@ -45,15 +45,16 @@ end function set_to_function!(u, f) # Supports serial and distributed - arch = child_architecture(u) + arch = architecture(u) + child_arch = child_architecture(u) # Determine cpu_grid and cpu_u - if arch isa GPU + if child_arch isa GPU cpu_arch = cpu_architecture(arch) cpu_grid = on_architecture(cpu_arch, u.grid) cpu_u = Field(location(u), cpu_grid; indices = indices(u)) - elseif arch isa CPU + elseif child_arch isa CPU cpu_grid = u.grid cpu_u = u end @@ -83,7 +84,7 @@ function set_to_function!(u, f) end # Transfer data to GPU if u is on the GPU - arch isa GPU && set!(u, cpu_u) + child_arch isa GPU && set!(u, cpu_u) return u end diff --git a/test/runtests.jl b/test/runtests.jl index 9c07f19b9a..219eb36452 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -172,6 +172,7 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() + # In case CUDA is not found, we reset CUDA and restart the julia session reset_cuda_if_necessary() archs = test_architectures() include("test_distributed_models.jl") @@ -179,6 +180,7 @@ CUDA.allowscalar() do if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() + # In case CUDA is not found, we reset CUDA and restart the julia session reset_cuda_if_necessary() include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") @@ -186,6 +188,7 @@ CUDA.allowscalar() do if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() + # In case CUDA is not found, we reset CUDA and restart the julia session reset_cuda_if_necessary() archs = test_architectures() include("test_hydrostatic_regression.jl") @@ -194,6 +197,7 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() + # In case CUDA is not found, we reset CUDA and restart the julia session reset_cuda_if_necessary() archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index 8dcf7fbf58..a027daf4de 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -8,6 +8,10 @@ import Oceananigans.Fields: interior child_arch = get(ENV, "GPU_TEST", nothing) == "true" ? GPU() : CPU() mpi_test = get(ENV, "MPI_TEST", nothing) == "true" +# Sometimes when running tests in parallel, the CUDA.jl package is not loaded correctly. +# This function is a failsafe to re-load CUDA.jl using the suggested cach compilation from +# https://github.com/JuliaGPU/CUDA.jl/blob/a085bbb3d7856dfa929e6cdae04a146a259a2044/src/initialization.jl#L105 +# To make sure Julia restarts, an error is thrown. function reset_cuda_if_necessary() # Do nothing if we are on the CPU @@ -18,12 +22,16 @@ function reset_cuda_if_necessary() try c = CUDA.zeros(10) # This will fail if CUDA is not available catch err + + # Avoid race conditions and precompile on rank 0 only if MPI.Comm_rank(MPI.COMM_WORLD) == 0 pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll") Base.compilecache(pkg) - @info "CUDA.jl was not loaded. Re-loading CUDA.jl and re-starting Julia." + @info "CUDA.jl was not correctly loaded. Re-loading CUDA.jl and re-starting Julia." end + MPI.Barrier(MPI.COMM_WORLD) + # re-start Julia and re-load CUDA.jl throw(err) end From 466ec0cd146781882f2f0b68ea68b415a6cd5bdd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 13:51:25 +0100 Subject: [PATCH 44/48] increase a bit the memory --- .buildkite/distributed/pipeline.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index c4e1d0cc57..23306e76d3 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -51,7 +51,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 8G + slurm_mem: 50G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -67,7 +67,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 8G + slurm_mem: 50G slurm_ntasks: 4 retry: automatic: @@ -83,7 +83,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 32G + slurm_mem: 50G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -99,7 +99,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 32G + slurm_mem: 50G slurm_ntasks: 4 retry: automatic: @@ -115,7 +115,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 32G + slurm_mem: 50G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -131,7 +131,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 32G + slurm_mem: 50G slurm_ntasks: 4 retry: automatic: @@ -147,7 +147,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 32G + slurm_mem: 50G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: From a27b383b8d0d9a92b9f55f8ac4f403e5db93d10f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 14:02:13 +0100 Subject: [PATCH 45/48] comment --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 23306e76d3..f9ba2d02d8 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,6 +1,6 @@ agents: queue: new-central - slurm_mem: 8G + slurm_mem: 8G # Note that the tests run on shared nodes, so limiting the memory usage might help in avoiding long queues modules: climacommon/2024_10_08 env: From eec18c2e3f3d630929c7509e33addc76473efbc2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 7 Nov 2024 14:02:35 +0100 Subject: [PATCH 46/48] whoops unit tests are small --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index f9ba2d02d8..84762749cb 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -51,7 +51,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 50G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: From 8011ef5b6424e4c825020a5c6f1306644293417a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 8 Nov 2024 12:05:02 +0100 Subject: [PATCH 47/48] increase memory limits --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 84762749cb..9f7869de63 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -115,7 +115,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 50G + slurm_mem: 80G # Apparently the GPU tests require more memory slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: From cd00381fad9ce42318266d7e9b0022b8247a4a5e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 8 Nov 2024 14:59:38 +0100 Subject: [PATCH 48/48] tests were running on the CPU on sverdrup --- .buildkite/pipeline.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1d6d6e31ae..49226230da 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -55,6 +55,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "unit" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -70,7 +71,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "unit" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -90,6 +90,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "poisson_solvers_1" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -105,7 +106,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "poisson_solvers_1" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -121,6 +121,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "poisson_solvers_2" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -136,7 +137,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "poisson_solvers_2" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -152,6 +152,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "matrix_poisson_solvers" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -167,7 +168,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "matrix_poisson_solvers" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -183,6 +183,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "general_solvers" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -198,7 +199,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "general_solvers" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -218,6 +218,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "time_stepping_1" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -233,7 +234,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "time_stepping_1" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -253,6 +253,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "time_stepping_2" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -268,7 +269,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "time_stepping_2" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -288,6 +288,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "time_stepping_3" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -303,7 +304,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "time_stepping_3" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -323,6 +323,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "turbulence_closures" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -338,7 +339,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "turbulence_closures" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -358,6 +358,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "hydrostatic_free_surface" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -373,7 +374,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "hydrostatic_free_surface" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -393,6 +393,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "shallow_water" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -408,7 +409,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "shallow_water" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -428,6 +428,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "simulation" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -443,7 +444,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "simulation" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -463,6 +463,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "lagrangian" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -478,7 +479,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "lagrangian" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -498,6 +498,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "abstract_operations" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -513,7 +514,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "abstract_operations" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -533,6 +533,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "multi_region" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -548,7 +549,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "multi_region" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -568,6 +568,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "nonhydrostatic_regression" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -583,7 +584,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "nonhydrostatic_regression" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -599,6 +599,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "hydrostatic_regression" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -614,7 +615,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "hydrostatic_regression" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -634,6 +634,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "scripts" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -649,7 +650,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "scripts" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -669,6 +669,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "enzyme" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -684,7 +685,6 @@ steps: env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "enzyme" - CUDA_VISIBLE_DEVICES: "-1" commands: - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: