Add Julia[up] UENV (#150)

deployed as `julia/24.9:v1` on todi and eiger
eth-cscs · Nov 9, 2024 · b82b452 · b82b452
1 parent b4dffe9
commit b82b452
Show file tree

Hide file tree

Showing 15 changed files with 391 additions and 1 deletion.
diff --git a/config.yaml b/config.yaml
@@ -52,7 +52,6 @@ clusters:
         partition: 'normal'
         variables:
           F7T_URL: "https://api.cscs.ch/cscs/firecrest/v1"
-          SLURM_RESERVATION: "daint"
     runner: f7t
 uenvs:
   arbor:
@@ -141,6 +140,13 @@ uenvs:
         todi: [gh200]
         santis: [gh200]
       develop: False
+  julia:
+    "24.9":
+      recipes:
+        zen2:  "24.9/mc"
+        zen3:  "24.9/mc"
+        gh200: "24.9/gh200"
+      develop: False
   lammps:
     "2023":
       recipes:

diff --git a/recipes/julia/24.9/gh200/compilers.yaml b/recipes/julia/24.9/gh200/compilers.yaml
@@ -0,0 +1,5 @@
+bootstrap:
+  spec: [email protected]
+gcc:
+  specs:
+  - [email protected]
diff --git a/recipes/julia/24.9/gh200/config.yaml b/recipes/julia/24.9/gh200/config.yaml
@@ -0,0 +1,7 @@
+name: julia
+spack:
+  commit: releases/v0.22
+  repo: https://github.com/spack/spack.git
+store: /user-environment
+description: HPC setup for juliaup, julia and some HPC key packages (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl, ADIOS2.jl, ...) + GNU Compiler toolchain with cray-mpich, HDF5, ADIOS2, Python, CMake and other development tools.
+
diff --git a/recipes/julia/24.9/gh200/environments.yaml b/recipes/julia/24.9/gh200/environments.yaml
@@ -0,0 +1,42 @@
+gcc-env:
+  compiler:
+      - toolchain: gcc
+        spec: gcc
+  mpi:
+      spec: [email protected]
+      gpu: cuda
+  unify: true
+  specs:
+  - aws-ofi-nccl@master
+  - cmake
+  - [email protected]
+  - fftw
+  - fmt
+  - hdf5+hl
+  - libtree
+  - meson
+  - nccl
+  - nccl-tests
+  - ninja
+  - openblas threads=openmp
+  - [email protected]
+  - python
+  - bzip2
+  - zfp
+  - sz
+  - libpng
+  - c-blosc2
+  - libpressio
+  - mgard
+  - libcatalyst
+  - [email protected]+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+libpressio+mgard+libcatalyst
+  variants:
+  - +mpi
+  - +cuda
+  - cuda_arch=90
+  views:
+    default:
+      link: roots
+      uenv:
+        prefix_paths:
+          LD_LIBRARY_PATH: [lib, lib64]
diff --git a/recipes/julia/24.9/gh200/extra/reframe.yaml b/recipes/julia/24.9/gh200/extra/reframe.yaml
@@ -0,0 +1,11 @@
+default:
+  features:
+    - cuda
+    - mpi
+    - osu-micro-benchmarks
+    - openmp
+    - serial
+  cc: mpicc
+  cxx: mpic++
+  ftn: mpifort
+  activation: /user-environment/env/default/activate.sh
diff --git a/recipes/julia/24.9/gh200/extra/uenv_view.jl b/recipes/julia/24.9/gh200/extra/uenv_view.jl
@@ -0,0 +1,78 @@
+# Author: Samuel Omlin, CSCS (omlins)   
+#
+# Description: Creation of an uenv view equivalent to the activation script in JUHPC.
+
+
+using Pkg; Pkg.add("JSON")
+using JSON
+
+
+# Check if the required environment variables are set
+
+function check_env(vars...)
+    for var in vars
+        if !haskey(ENV, var)
+            error("uenv-view: $var is not set.")
+        elseif isempty(ENV[var])
+            error("uenv-view: $var is empty.")
+        end
+    end
+end
+
+check_env("JULIAUP_WRAPPER_BINDIR", "JULIAUP_BINDIR", "JULIAUP_DEPOT", "JULIA_DEPOT", "JULIA_PREFDIR", "JULIAUP_INSTALLDIR", "ENV_JSON")
+
+
+# Define the environment variables part of the julia view
+
+julia_view_env = Dict(
+    "values" => Dict(
+        "list" => Dict(
+            "PATH" => [
+                Dict(
+                    "value" => [ENV["JULIAUP_WRAPPER_BINDIR"], ENV["JULIAUP_BINDIR"]], # The wrapper must be before the juliaup bindir
+                    "op" => "prepend",
+                ),
+            ],
+        ),
+        "scalar" => Dict(
+            "JULIAUP_DEPOT_PATH" => ENV["JULIAUP_DEPOT"],
+            "JULIA_DEPOT_PATH"   => ENV["JULIA_DEPOT"],
+            "JULIA_LOAD_PATH"    => ":$(ENV["JULIA_PREFDIR"])", # ":" means appending!
+            [key => ENV[env_key] for (key, env_key) in 
+                [("CUDA_HOME", "JUHPC_CUDA_HOME"), 
+                 ("ROCM_PATH", "JUHPC_ROCM_HOME"), 
+                 ("JULIA_ADIOS2_PATH", "JUHPC_ADIOS2_HOME")]
+                if haskey(ENV, env_key) && !isempty(ENV[env_key])]..., # Conditional inclusion for variables if they are set and not empty
+            [("JULIA_CUDA_MEMORY_POOL" => "none") for env_key in ["JUHPC_CUDA_HOME"] if haskey(ENV, env_key) && !isempty(ENV[env_key])]... # Conditionally include JULIA_CUDA_MEMORY_POOL
+        )
+    ),
+    "version" => 1
+)
+
+
+# Define the julia view
+
+julia_view = Dict(
+    "julia" => Dict(
+        "root" => "/user-environment/env/julia",
+        "env" => julia_view_env,
+        "activate" => "/dev/null",
+        "description" => "description: HPC setup for juliaup, julia and some HPC key packages (juliaup and julia are installed on first execution of juliaup in $(ENV["JULIAUP_INSTALLDIR"]))",
+        "type" => "augment"
+    )
+)
+
+
+# Merge the julia view with the existing views in the JSON file
+
+env = JSON.parsefile(ENV["ENV_JSON"])
+views = env["views"]
+views = merge(views, julia_view)
+env["views"] = views
+open(ENV["ENV_JSON"],"w") do f
+    JSON.print(f, env, 4)
+end
+
+
+# Remove the added package
+Pkg.rm("JSON")
diff --git a/recipes/julia/24.9/gh200/modules.yaml b/recipes/julia/24.9/gh200/modules.yaml
@@ -0,0 +1,23 @@
+modules:
+  # Paths to check when creating modules for all module sets
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    # Where to install modules
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%[email protected]', 'gcc %[email protected]']
+      projections:
+        all: '{name}/{version}'
diff --git a/recipes/julia/24.9/gh200/post-install b/recipes/julia/24.9/gh200/post-install
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Author: Samuel Omlin, CSCS (omlins)
+#
+# Description: Definition of site specific variables and call of JUHPC.
+# Site: ALPS:todi, Swiss National Supercomputing Centre (CSCS)
+# Base: uenv
+
+
+# UENV specific environment variables
+export ENV_MOUNT={{ env.mount }}    # export ENV_MOUNT=/user-environment
+export ENV_META=$ENV_MOUNT/meta
+export ENV_EXTRA=$ENV_META/extra
+export ENV_JSON=$ENV_META/env.json
+
+
+# Environment variables for HPC key packages that require system libraries (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl and ADIOS2.jl)
+export JUHPC_CUDA_HOME=$(spack -C $ENV_MOUNT/config location -i cuda)
+export JUHPC_CUDA_RUNTIME_VERSION=$(spack --color=never -C $ENV_MOUNT/config find cuda | \
+                                    perl -ne 'print $1 if /cuda@([\d.]+)/')
+export JUHPC_MPI_HOME=$(spack -C $ENV_MOUNT/config location -i cray-mpich)
+export JUHPC_MPI_EXEC="srun -C gpu"
+export JUHPC_HDF5_HOME=$(spack -C $ENV_MOUNT/config location -i hdf5)
+export JUHPC_ADIOS2_HOME=$(spack -C $ENV_MOUNT/config location -i adios2)
+
+
+# Call JUHPC
+JUHPC_SETUP_INSTALLDIR=$ENV_MOUNT/juhpc_setup
+JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup"
+JUHPC_POST_INSTALL_JL=$ENV_EXTRA/uenv_view.jl
+VERSION="v0.2.0"
+wget https://raw.githubusercontent.com/JuliaParallel/JUHPC/$VERSION/juhpc -O /tmp/juhpc
+bash -l /tmp/juhpc $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR $JUHPC_POST_INSTALL_JL
diff --git a/recipes/julia/24.9/mc/compilers.yaml b/recipes/julia/24.9/mc/compilers.yaml
@@ -0,0 +1,5 @@
+bootstrap:
+  spec: gcc@11
+gcc:
+  specs:
+  - gcc@13
diff --git a/recipes/julia/24.9/mc/config.yaml b/recipes/julia/24.9/mc/config.yaml
@@ -0,0 +1,7 @@
+name: julia
+spack:
+  commit: releases/v0.22
+  repo: https://github.com/spack/spack.git
+store: /user-environment
+description: HPC setup for juliaup, julia and some HPC key packages (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl, ADIOS2.jl, ...) + GNU Compiler toolchain with cray-mpich, HDF5, ADIOS2, Python, CMake and other development tools.
+
diff --git a/recipes/julia/24.9/mc/environments.yaml b/recipes/julia/24.9/mc/environments.yaml
@@ -0,0 +1,36 @@
+gcc-env:
+  compiler:
+      - toolchain: gcc
+        spec: gcc@13
+  mpi:
+      spec: cray-mpich
+      gpu: Null
+  unify: true
+  specs:
+  - cmake
+  - fftw
+  - fmt
+  - hdf5+hl
+  - [email protected]
+  - openblas threads=openmp
+  - python
+  - py-pybind11
+  - [email protected]
+  - bzip2
+  - zfp
+  - sz
+  - libpng
+  - c-blosc2
+  - libpressio
+  - mgard
+  - libcatalyst
+  - [email protected]+hdf5+python+fortran+sst+pic+shared+bzip2+zfp+sz+png+blosc2+libpressio+mgard+libcatalyst
+  variants:
+  - +mpi
+  views:
+    develop:
+      link: roots
+      uenv:
+        add_compilers: true
+        prefix_paths:
+          LD_LIBRARY_PATH: [lib, lib64]
diff --git a/recipes/julia/24.9/mc/extra/reframe.yaml b/recipes/julia/24.9/mc/extra/reframe.yaml
@@ -0,0 +1,6 @@
+default:
+  features: [osu-micro-benchmarks, mpi, serial, openmp]
+  cc: mpicc
+  cxx: mpic++
+  ftn: mpifort
+  activation: /user-environment/env/default/activate.sh
diff --git a/recipes/julia/24.9/mc/extra/uenv_view.jl b/recipes/julia/24.9/mc/extra/uenv_view.jl
@@ -0,0 +1,78 @@
+# Author: Samuel Omlin, CSCS (omlins)   
+#
+# Description: Creation of an uenv view equivalent to the activation script in JUHPC.
+
+
+using Pkg; Pkg.add("JSON")
+using JSON
+
+
+# Check if the required environment variables are set
+
+function check_env(vars...)
+    for var in vars
+        if !haskey(ENV, var)
+            error("uenv-view: $var is not set.")
+        elseif isempty(ENV[var])
+            error("uenv-view: $var is empty.")
+        end
+    end
+end
+
+check_env("JULIAUP_WRAPPER_BINDIR", "JULIAUP_BINDIR", "JULIAUP_DEPOT", "JULIA_DEPOT", "JULIA_PREFDIR", "JULIAUP_INSTALLDIR", "ENV_JSON")
+
+
+# Define the environment variables part of the julia view
+
+julia_view_env = Dict(
+    "values" => Dict(
+        "list" => Dict(
+            "PATH" => [
+                Dict(
+                    "value" => [ENV["JULIAUP_WRAPPER_BINDIR"], ENV["JULIAUP_BINDIR"]], # The wrapper must be before the juliaup bindir
+                    "op" => "prepend",
+                ),
+            ],
+        ),
+        "scalar" => Dict(
+            "JULIAUP_DEPOT_PATH" => ENV["JULIAUP_DEPOT"],
+            "JULIA_DEPOT_PATH"   => ENV["JULIA_DEPOT"],
+            "JULIA_LOAD_PATH"    => ":$(ENV["JULIA_PREFDIR"])", # ":" means appending!
+            [key => ENV[env_key] for (key, env_key) in 
+                [("CUDA_HOME", "JUHPC_CUDA_HOME"), 
+                 ("ROCM_PATH", "JUHPC_ROCM_HOME"), 
+                 ("JULIA_ADIOS2_PATH", "JUHPC_ADIOS2_HOME")]
+                if haskey(ENV, env_key) && !isempty(ENV[env_key])]..., # Conditional inclusion for variables if they are set and not empty
+            [("JULIA_CUDA_MEMORY_POOL" => "none") for env_key in ["JUHPC_CUDA_HOME"] if haskey(ENV, env_key) && !isempty(ENV[env_key])]... # Conditionally include JULIA_CUDA_MEMORY_POOL
+        )
+    ),
+    "version" => 1
+)
+
+
+# Define the julia view
+
+julia_view = Dict(
+    "julia" => Dict(
+        "root" => "/user-environment/env/julia",
+        "env" => julia_view_env,
+        "activate" => "/dev/null",
+        "description" => "description: HPC setup for juliaup, julia and some HPC key packages (juliaup and julia are installed on first execution of juliaup in $(ENV["JULIAUP_INSTALLDIR"]))",
+        "type" => "augment"
+    )
+)
+
+
+# Merge the julia view with the existing views in the JSON file
+
+env = JSON.parsefile(ENV["ENV_JSON"])
+views = env["views"]
+views = merge(views, julia_view)
+env["views"] = views
+open(ENV["ENV_JSON"],"w") do f
+    JSON.print(f, env, 4)
+end
+
+
+# Remove the added package
+Pkg.rm("JSON")
diff --git a/recipes/julia/24.9/mc/modules.yaml b/recipes/julia/24.9/mc/modules.yaml
@@ -0,0 +1,23 @@
+modules:
+  # Paths to check when creating modules for all module sets
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    # Where to install modules
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%[email protected]', 'gcc %[email protected]']
+      projections:
+        all: '{name}/{version}'