From b82b452def804200d654ca2e8bb5ff3c48c81989 Mon Sep 17 00:00:00 2001 From: Samuel Omlin Date: Sat, 9 Nov 2024 05:57:29 +0100 Subject: [PATCH] Add Julia[up] UENV (#150) deployed as `julia/24.9:v1` on todi and eiger --- config.yaml | 8 ++- recipes/julia/24.9/gh200/compilers.yaml | 5 ++ recipes/julia/24.9/gh200/config.yaml | 7 ++ recipes/julia/24.9/gh200/environments.yaml | 42 +++++++++++ recipes/julia/24.9/gh200/extra/reframe.yaml | 11 +++ recipes/julia/24.9/gh200/extra/uenv_view.jl | 78 +++++++++++++++++++++ recipes/julia/24.9/gh200/modules.yaml | 23 ++++++ recipes/julia/24.9/gh200/post-install | 33 +++++++++ recipes/julia/24.9/mc/compilers.yaml | 5 ++ recipes/julia/24.9/mc/config.yaml | 7 ++ recipes/julia/24.9/mc/environments.yaml | 36 ++++++++++ recipes/julia/24.9/mc/extra/reframe.yaml | 6 ++ recipes/julia/24.9/mc/extra/uenv_view.jl | 78 +++++++++++++++++++++ recipes/julia/24.9/mc/modules.yaml | 23 ++++++ recipes/julia/24.9/mc/post-install | 30 ++++++++ 15 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 recipes/julia/24.9/gh200/compilers.yaml create mode 100644 recipes/julia/24.9/gh200/config.yaml create mode 100644 recipes/julia/24.9/gh200/environments.yaml create mode 100644 recipes/julia/24.9/gh200/extra/reframe.yaml create mode 100644 recipes/julia/24.9/gh200/extra/uenv_view.jl create mode 100644 recipes/julia/24.9/gh200/modules.yaml create mode 100644 recipes/julia/24.9/gh200/post-install create mode 100644 recipes/julia/24.9/mc/compilers.yaml create mode 100644 recipes/julia/24.9/mc/config.yaml create mode 100644 recipes/julia/24.9/mc/environments.yaml create mode 100644 recipes/julia/24.9/mc/extra/reframe.yaml create mode 100644 recipes/julia/24.9/mc/extra/uenv_view.jl create mode 100644 recipes/julia/24.9/mc/modules.yaml create mode 100644 recipes/julia/24.9/mc/post-install diff --git a/config.yaml b/config.yaml index d1cc818b..17f92c39 100644 --- a/config.yaml +++ b/config.yaml @@ -52,7 +52,6 @@ clusters: partition: 'normal' variables: F7T_URL: "https://api.cscs.ch/cscs/firecrest/v1" - SLURM_RESERVATION: "daint" runner: f7t uenvs: arbor: @@ -141,6 +140,13 @@ uenvs: todi: [gh200] santis: [gh200] develop: False + julia: + "24.9": + recipes: + zen2: "24.9/mc" + zen3: "24.9/mc" + gh200: "24.9/gh200" + develop: False lammps: "2023": recipes: diff --git a/recipes/julia/24.9/gh200/compilers.yaml b/recipes/julia/24.9/gh200/compilers.yaml new file mode 100644 index 00000000..8209f0c2 --- /dev/null +++ b/recipes/julia/24.9/gh200/compilers.yaml @@ -0,0 +1,5 @@ +bootstrap: + spec: gcc@12.3 +gcc: + specs: + - gcc@13.2 diff --git a/recipes/julia/24.9/gh200/config.yaml b/recipes/julia/24.9/gh200/config.yaml new file mode 100644 index 00000000..176d2a8c --- /dev/null +++ b/recipes/julia/24.9/gh200/config.yaml @@ -0,0 +1,7 @@ +name: julia +spack: + commit: releases/v0.22 + repo: https://github.com/spack/spack.git +store: /user-environment +description: HPC setup for juliaup, julia and some HPC key packages (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl, ADIOS2.jl, ...) + GNU Compiler toolchain with cray-mpich, HDF5, ADIOS2, Python, CMake and other development tools. + diff --git a/recipes/julia/24.9/gh200/environments.yaml b/recipes/julia/24.9/gh200/environments.yaml new file mode 100644 index 00000000..24760579 --- /dev/null +++ b/recipes/julia/24.9/gh200/environments.yaml @@ -0,0 +1,42 @@ +gcc-env: + compiler: + - toolchain: gcc + spec: gcc + mpi: + spec: cray-mpich@8.1.30 + gpu: cuda + unify: true + specs: + - aws-ofi-nccl@master + - cmake + - cuda@12.4 + - fftw + - fmt + - hdf5+hl + - libtree + - meson + - nccl + - nccl-tests + - ninja + - openblas threads=openmp + - osu-micro-benchmarks@5.9 + - python + - bzip2 + - zfp + - sz + - libpng + - c-blosc2 + - libpressio + - mgard + - libcatalyst + - adios2@2.10+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+libpressio+mgard+libcatalyst + variants: + - +mpi + - +cuda + - cuda_arch=90 + views: + default: + link: roots + uenv: + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] diff --git a/recipes/julia/24.9/gh200/extra/reframe.yaml b/recipes/julia/24.9/gh200/extra/reframe.yaml new file mode 100644 index 00000000..b71f2797 --- /dev/null +++ b/recipes/julia/24.9/gh200/extra/reframe.yaml @@ -0,0 +1,11 @@ +default: + features: + - cuda + - mpi + - osu-micro-benchmarks + - openmp + - serial + cc: mpicc + cxx: mpic++ + ftn: mpifort + activation: /user-environment/env/default/activate.sh diff --git a/recipes/julia/24.9/gh200/extra/uenv_view.jl b/recipes/julia/24.9/gh200/extra/uenv_view.jl new file mode 100644 index 00000000..229a0ec5 --- /dev/null +++ b/recipes/julia/24.9/gh200/extra/uenv_view.jl @@ -0,0 +1,78 @@ +# Author: Samuel Omlin, CSCS (omlins) +# +# Description: Creation of an uenv view equivalent to the activation script in JUHPC. + + +using Pkg; Pkg.add("JSON") +using JSON + + +# Check if the required environment variables are set + +function check_env(vars...) + for var in vars + if !haskey(ENV, var) + error("uenv-view: $var is not set.") + elseif isempty(ENV[var]) + error("uenv-view: $var is empty.") + end + end +end + +check_env("JULIAUP_WRAPPER_BINDIR", "JULIAUP_BINDIR", "JULIAUP_DEPOT", "JULIA_DEPOT", "JULIA_PREFDIR", "JULIAUP_INSTALLDIR", "ENV_JSON") + + +# Define the environment variables part of the julia view + +julia_view_env = Dict( + "values" => Dict( + "list" => Dict( + "PATH" => [ + Dict( + "value" => [ENV["JULIAUP_WRAPPER_BINDIR"], ENV["JULIAUP_BINDIR"]], # The wrapper must be before the juliaup bindir + "op" => "prepend", + ), + ], + ), + "scalar" => Dict( + "JULIAUP_DEPOT_PATH" => ENV["JULIAUP_DEPOT"], + "JULIA_DEPOT_PATH" => ENV["JULIA_DEPOT"], + "JULIA_LOAD_PATH" => ":$(ENV["JULIA_PREFDIR"])", # ":" means appending! + [key => ENV[env_key] for (key, env_key) in + [("CUDA_HOME", "JUHPC_CUDA_HOME"), + ("ROCM_PATH", "JUHPC_ROCM_HOME"), + ("JULIA_ADIOS2_PATH", "JUHPC_ADIOS2_HOME")] + if haskey(ENV, env_key) && !isempty(ENV[env_key])]..., # Conditional inclusion for variables if they are set and not empty + [("JULIA_CUDA_MEMORY_POOL" => "none") for env_key in ["JUHPC_CUDA_HOME"] if haskey(ENV, env_key) && !isempty(ENV[env_key])]... # Conditionally include JULIA_CUDA_MEMORY_POOL + ) + ), + "version" => 1 +) + + +# Define the julia view + +julia_view = Dict( + "julia" => Dict( + "root" => "/user-environment/env/julia", + "env" => julia_view_env, + "activate" => "/dev/null", + "description" => "description: HPC setup for juliaup, julia and some HPC key packages (juliaup and julia are installed on first execution of juliaup in $(ENV["JULIAUP_INSTALLDIR"]))", + "type" => "augment" + ) +) + + +# Merge the julia view with the existing views in the JSON file + +env = JSON.parsefile(ENV["ENV_JSON"]) +views = env["views"] +views = merge(views, julia_view) +env["views"] = views +open(ENV["ENV_JSON"],"w") do f + JSON.print(f, env, 4) +end + + +# Remove the added package +Pkg.rm("JSON") \ No newline at end of file diff --git a/recipes/julia/24.9/gh200/modules.yaml b/recipes/julia/24.9/gh200/modules.yaml new file mode 100644 index 00000000..623307b0 --- /dev/null +++ b/recipes/julia/24.9/gh200/modules.yaml @@ -0,0 +1,23 @@ +modules: + # Paths to check when creating modules for all module sets + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + # Where to install modules + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + projections: + all: '{name}/{version}' diff --git a/recipes/julia/24.9/gh200/post-install b/recipes/julia/24.9/gh200/post-install new file mode 100644 index 00000000..76cc9b3d --- /dev/null +++ b/recipes/julia/24.9/gh200/post-install @@ -0,0 +1,33 @@ +#!/bin/bash + +# Author: Samuel Omlin, CSCS (omlins) +# +# Description: Definition of site specific variables and call of JUHPC. +# Site: ALPS:todi, Swiss National Supercomputing Centre (CSCS) +# Base: uenv + + +# UENV specific environment variables +export ENV_MOUNT={{ env.mount }} # export ENV_MOUNT=/user-environment +export ENV_META=$ENV_MOUNT/meta +export ENV_EXTRA=$ENV_META/extra +export ENV_JSON=$ENV_META/env.json + + +# Environment variables for HPC key packages that require system libraries (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl and ADIOS2.jl) +export JUHPC_CUDA_HOME=$(spack -C $ENV_MOUNT/config location -i cuda) +export JUHPC_CUDA_RUNTIME_VERSION=$(spack --color=never -C $ENV_MOUNT/config find cuda | \ + perl -ne 'print $1 if /cuda@([\d.]+)/') +export JUHPC_MPI_HOME=$(spack -C $ENV_MOUNT/config location -i cray-mpich) +export JUHPC_MPI_EXEC="srun -C gpu" +export JUHPC_HDF5_HOME=$(spack -C $ENV_MOUNT/config location -i hdf5) +export JUHPC_ADIOS2_HOME=$(spack -C $ENV_MOUNT/config location -i adios2) + + +# Call JUHPC +JUHPC_SETUP_INSTALLDIR=$ENV_MOUNT/juhpc_setup +JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup" +JUHPC_POST_INSTALL_JL=$ENV_EXTRA/uenv_view.jl +VERSION="v0.2.0" +wget https://raw.githubusercontent.com/JuliaParallel/JUHPC/$VERSION/juhpc -O /tmp/juhpc +bash -l /tmp/juhpc $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR $JUHPC_POST_INSTALL_JL diff --git a/recipes/julia/24.9/mc/compilers.yaml b/recipes/julia/24.9/mc/compilers.yaml new file mode 100644 index 00000000..bbe4e268 --- /dev/null +++ b/recipes/julia/24.9/mc/compilers.yaml @@ -0,0 +1,5 @@ +bootstrap: + spec: gcc@11 +gcc: + specs: + - gcc@13 diff --git a/recipes/julia/24.9/mc/config.yaml b/recipes/julia/24.9/mc/config.yaml new file mode 100644 index 00000000..176d2a8c --- /dev/null +++ b/recipes/julia/24.9/mc/config.yaml @@ -0,0 +1,7 @@ +name: julia +spack: + commit: releases/v0.22 + repo: https://github.com/spack/spack.git +store: /user-environment +description: HPC setup for juliaup, julia and some HPC key packages (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl, ADIOS2.jl, ...) + GNU Compiler toolchain with cray-mpich, HDF5, ADIOS2, Python, CMake and other development tools. + diff --git a/recipes/julia/24.9/mc/environments.yaml b/recipes/julia/24.9/mc/environments.yaml new file mode 100644 index 00000000..247d1ab9 --- /dev/null +++ b/recipes/julia/24.9/mc/environments.yaml @@ -0,0 +1,36 @@ +gcc-env: + compiler: + - toolchain: gcc + spec: gcc@13 + mpi: + spec: cray-mpich + gpu: Null + unify: true + specs: + - cmake + - fftw + - fmt + - hdf5+hl + - ninja@1.11 + - openblas threads=openmp + - python + - py-pybind11 + - osu-micro-benchmarks@5.9 + - bzip2 + - zfp + - sz + - libpng + - c-blosc2 + - libpressio + - mgard + - libcatalyst + - adios2@2.10+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+libpressio+mgard+libcatalyst + variants: + - +mpi + views: + develop: + link: roots + uenv: + add_compilers: true + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] diff --git a/recipes/julia/24.9/mc/extra/reframe.yaml b/recipes/julia/24.9/mc/extra/reframe.yaml new file mode 100644 index 00000000..363f809f --- /dev/null +++ b/recipes/julia/24.9/mc/extra/reframe.yaml @@ -0,0 +1,6 @@ +default: + features: [osu-micro-benchmarks, mpi, serial, openmp] + cc: mpicc + cxx: mpic++ + ftn: mpifort + activation: /user-environment/env/default/activate.sh diff --git a/recipes/julia/24.9/mc/extra/uenv_view.jl b/recipes/julia/24.9/mc/extra/uenv_view.jl new file mode 100644 index 00000000..229a0ec5 --- /dev/null +++ b/recipes/julia/24.9/mc/extra/uenv_view.jl @@ -0,0 +1,78 @@ +# Author: Samuel Omlin, CSCS (omlins) +# +# Description: Creation of an uenv view equivalent to the activation script in JUHPC. + + +using Pkg; Pkg.add("JSON") +using JSON + + +# Check if the required environment variables are set + +function check_env(vars...) + for var in vars + if !haskey(ENV, var) + error("uenv-view: $var is not set.") + elseif isempty(ENV[var]) + error("uenv-view: $var is empty.") + end + end +end + +check_env("JULIAUP_WRAPPER_BINDIR", "JULIAUP_BINDIR", "JULIAUP_DEPOT", "JULIA_DEPOT", "JULIA_PREFDIR", "JULIAUP_INSTALLDIR", "ENV_JSON") + + +# Define the environment variables part of the julia view + +julia_view_env = Dict( + "values" => Dict( + "list" => Dict( + "PATH" => [ + Dict( + "value" => [ENV["JULIAUP_WRAPPER_BINDIR"], ENV["JULIAUP_BINDIR"]], # The wrapper must be before the juliaup bindir + "op" => "prepend", + ), + ], + ), + "scalar" => Dict( + "JULIAUP_DEPOT_PATH" => ENV["JULIAUP_DEPOT"], + "JULIA_DEPOT_PATH" => ENV["JULIA_DEPOT"], + "JULIA_LOAD_PATH" => ":$(ENV["JULIA_PREFDIR"])", # ":" means appending! + [key => ENV[env_key] for (key, env_key) in + [("CUDA_HOME", "JUHPC_CUDA_HOME"), + ("ROCM_PATH", "JUHPC_ROCM_HOME"), + ("JULIA_ADIOS2_PATH", "JUHPC_ADIOS2_HOME")] + if haskey(ENV, env_key) && !isempty(ENV[env_key])]..., # Conditional inclusion for variables if they are set and not empty + [("JULIA_CUDA_MEMORY_POOL" => "none") for env_key in ["JUHPC_CUDA_HOME"] if haskey(ENV, env_key) && !isempty(ENV[env_key])]... # Conditionally include JULIA_CUDA_MEMORY_POOL + ) + ), + "version" => 1 +) + + +# Define the julia view + +julia_view = Dict( + "julia" => Dict( + "root" => "/user-environment/env/julia", + "env" => julia_view_env, + "activate" => "/dev/null", + "description" => "description: HPC setup for juliaup, julia and some HPC key packages (juliaup and julia are installed on first execution of juliaup in $(ENV["JULIAUP_INSTALLDIR"]))", + "type" => "augment" + ) +) + + +# Merge the julia view with the existing views in the JSON file + +env = JSON.parsefile(ENV["ENV_JSON"]) +views = env["views"] +views = merge(views, julia_view) +env["views"] = views +open(ENV["ENV_JSON"],"w") do f + JSON.print(f, env, 4) +end + + +# Remove the added package +Pkg.rm("JSON") \ No newline at end of file diff --git a/recipes/julia/24.9/mc/modules.yaml b/recipes/julia/24.9/mc/modules.yaml new file mode 100644 index 00000000..623307b0 --- /dev/null +++ b/recipes/julia/24.9/mc/modules.yaml @@ -0,0 +1,23 @@ +modules: + # Paths to check when creating modules for all module sets + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + # Where to install modules + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + projections: + all: '{name}/{version}' diff --git a/recipes/julia/24.9/mc/post-install b/recipes/julia/24.9/mc/post-install new file mode 100644 index 00000000..c5357ca8 --- /dev/null +++ b/recipes/julia/24.9/mc/post-install @@ -0,0 +1,30 @@ +#!/bin/bash + +# Author: Samuel Omlin, CSCS (omlins) +# +# Description: Definition of site specific variables and call of JUHPC. +# Site: ALPS:eiger, Swiss National Supercomputing Centre (CSCS) +# Base: uenv + + +# UENV specific environment variables +export ENV_MOUNT={{ env.mount }} # export ENV_MOUNT=/user-environment +export ENV_META=$ENV_MOUNT/meta +export ENV_EXTRA=$ENV_META/extra +export ENV_JSON=$ENV_META/env.json + + +# Environment variables for HPC key packages that require system libraries (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl and ADIOS2.jl) +export JUHPC_MPI_HOME=$(spack -C $ENV_MOUNT/config location -i cray-mpich) +export JUHPC_MPI_EXEC="srun -C mc" +export JUHPC_HDF5_HOME=$(spack -C $ENV_MOUNT/config location -i hdf5) +export JUHPC_ADIOS2_HOME=$(spack -C $ENV_MOUNT/config location -i adios2) + + +# Call JUHPC +JUHPC_SETUP_INSTALLDIR=$ENV_MOUNT/juhpc_setup +JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup" +JUHPC_POST_INSTALL_JL=$ENV_EXTRA/uenv_view.jl +VERSION="v0.2.0" +wget https://raw.githubusercontent.com/JuliaParallel/JUHPC/$VERSION/juhpc -O /tmp/juhpc +bash -l /tmp/juhpc $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR $JUHPC_POST_INSTALL_JL