CP2K 2024.3 and F7T runner for Todi (#148)

Users hit a bug in CP2K affecting MD and cell relaxation, that has been fixed in 2024.3. cp2k/cp2k#3661
eth-cscs · Sep 18, 2024 · 521487d · 521487d
1 parent 52a6bb6
commit 521487d
Show file tree

Hide file tree

Showing 40 changed files with 5,819 additions and 12 deletions.
diff --git a/config.yaml b/config.yaml
@@ -16,16 +16,8 @@ clusters:
         uarch: 'gh200'
         partition: 'normal'
         variables:
-          # new daint has a custom url (temporary?)
-          F7T_URL: "https://firecrest-hpc.v1.svc.cscs.ch"
+          F7T_URL: "https://api.cscs.ch/hpc/firecrest/v1"
     runner: f7t
-    #targets:
-    #  -
-    #    uarch: 'gh200'
-    #    partition: 'normal'
-    #runner:
-    #  slurm-tag:  daint-spack-stack-builder
-    #  baremetal-tag: daint-login-baremetal
   eiger:
     targets:
       -
@@ -57,9 +49,9 @@ clusters:
       -
         uarch: 'gh200'
         partition: 'normal'
-    runner:
-      slurm-tag:  todi-spack-stack-builder
-      baremetal-tag: todi-login-baremetal
+        variables:
+          F7T_URL: "https://api.cscs.ch/cscs/firecrest/v1"
+    runner: f7t
 uenvs:
   arbor:
     "v0.9":
@@ -120,6 +112,10 @@ uenvs:
       recipes:
         gh200: 2024.2/gh200
         zen2: 2024.2/mc
+    "2024.3":
+      recipes:
+        gh200: 2024.3/gh200
+        zen2: 2024.3/mc
       deploy:
         daint: [gh200]
         todi: [gh200]

diff --git a/recipes/cp2k/2024.3/gh200/compilers.yaml b/recipes/cp2k/2024.3/gh200/compilers.yaml
@@ -0,0 +1,5 @@
+bootstrap:
+  spec: gcc@12
+gcc:
+  specs:
+  - [email protected]
diff --git a/recipes/cp2k/2024.3/gh200/config.yaml b/recipes/cp2k/2024.3/gh200/config.yaml
@@ -0,0 +1,6 @@
+name: cp2k
+store: /user-environment
+spack:
+  repo: https://github.com/spack/spack.git
+  commit: v0.22.1
+modules: True
diff --git a/recipes/cp2k/2024.3/gh200/environments.yaml b/recipes/cp2k/2024.3/gh200/environments.yaml
@@ -0,0 +1,44 @@
+cp2k:
+  compiler:
+  - toolchain: gcc
+    spec: [email protected]
+  unify: true
+  specs:
+  - cmake
+  - ninja
+  - [email protected] # Can't be newer than driver version, for DBCSR
+  - openblas threads=openmp
+  - netlib-scalapack
+  - fftw
+  - libxc ~cuda
+  - libint
+  - spglib
+  - spla
+  - elpa
+  - plumed
+  # COSMA
+  - tiled-mm
+  - costa
+  - cosma +gpu_direct
+  # SIRIUS
+  - hdf5
+  - spfft
+  - sirius ~memory_pool
+  # CP2K
+  - [email protected]
+  - [email protected] +libxc +libint +spglib +cosma +spla +elpa +sirius +plumed lmax=5 build_system=cmake
+  variants:
+    - +mpi
+    - +openmp
+    - +cuda
+    - cuda_arch=90
+    - build_type=Release
+  mpi:
+    spec: [email protected]
+    gpu: cuda
+  views:
+    develop:
+      link: roots
+      exclude: ["cp2k"]
+    cp2k:
+      link: roots
diff --git a/recipes/cp2k/2024.3/gh200/modules.yaml b/recipes/cp2k/2024.3/gh200/modules.yaml
@@ -0,0 +1,21 @@
+modules:
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%[email protected]', 'gcc %[email protected]']
+      projections:
+        all: '{name}/{version}'
diff --git a/recipes/cp2k/2024.3/gh200/repo/packages/cosma/fj-ssl2.patch b/recipes/cp2k/2024.3/gh200/repo/packages/cosma/fj-ssl2.patch
@@ -0,0 +1,104 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 1fd1e55..41a041b 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -19,7 +19,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS "YES") # always write compile_commands.json
+
+ set(COSMA_GPU_BACKENDS_LIST "CUDA" "ROCM")
+ set(COSMA_SCALAPACK_LIST "OFF" "MKL" "CRAY_LIBSCI" "CUSTOM")
+-set(COSMA_BLAS_LIST   "auto" "MKL" "OPENBLAS" "CRAY_LIBSCI" "CUSTOM" "BLIS" "ATLAS" "CUDA" "ROCM" "OFF")
++set(COSMA_BLAS_LIST   "auto" "MKL" "SSL2" "OPENBLAS" "CRAY_LIBSCI" "CUSTOM" "BLIS" "ATLAS" "CUDA" "ROCM" "OFF")
+ option(COSMA_WITH_TESTS "Generate the test target." ON)
+ option(COSMA_WITH_APPS "Generate the miniapp targets." ON)
+ option(COSMA_WITH_BENCHMARKS "Generate the benchmark targets." ON)
+@@ -45,7 +45,7 @@ if (COSMA_BLAS MATCHES "CUDA|ROCM")
+   set(COSMA_GPU_BACKEND ${COSMA_BLAS})
+ else()
+   if(COSMA_BLAS STREQUAL "OFF")
+-    message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, OPENBLAS, CRAY_LIBSCI, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS")
++    message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, SSL2, OPENBLAS, CRAY_LIBSCI, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS")
+   else()
+     set(COSMA_BLAS_VENDOR ${COSMA_BLAS})
+   endif()
+@@ -190,6 +190,7 @@ install(FILES "${cosma_BINARY_DIR}/cosmaConfig.cmake"
+   "${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
+   "${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindMKL.cmake"
++  "${cosma_SOURCE_DIR}/cmake/FindSSL2.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindBlas.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindSCALAPACK.cmake"
+   "${cosma_SOURCE_DIR}/cmake/FindOPENBLAS.cmake"
+diff --git a/cmake/FindBlas.cmake b/cmake/FindBlas.cmake
+index aef956c..3c47561 100644
+--- a/cmake/FindBlas.cmake
++++ b/cmake/FindBlas.cmake
+@@ -14,6 +14,7 @@ endif()
+ set(COSMA_BLAS_VENDOR_LIST
+   "auto"
+   "MKL"
++  "SSL2"
+   "OPENBLAS"
+   "FLEXIBLAS"
+   "ARMPL"
+diff --git a/cmake/FindSSL2.cmake b/cmake/FindSSL2.cmake
+new file mode 100644
+index 0000000..f0e11bf
+--- /dev/null
++++ b/cmake/FindSSL2.cmake
+@@ -0,0 +1,56 @@
++#.rst:
++# FindSSL2
++# -----------
++#
++# This module tries to find the SSL2 library.
++#
++# The following variables are set
++#
++# ::
++#
++#   SSL2_FOUND           - True if ssl2 is found
++#   SSL2_LIBRARIES       - The required libraries
++#   SSL2_INCLUDE_DIRS    - The required include directory
++#
++# The following import target is created
++#
++# ::
++#
++#   SSL2::ssl2
++
++#set paths to look for library from ROOT variables.If new policy is set, find_library() automatically uses them.
++# if(NOT POLICY CMP0074)
++set(_SSL2_PATHS ${SSL2_ROOT}
++                 $ENV{SSL2_ROOT}
++                 $ENV{SSL2ROOT}
++                 $ENV{SSL2_DIR}
++                 $ENV{SSL2DIR})
++# endif()
++
++find_library(
++    COSMA_SSL2_LINK_LIBRARIES
++    NAMES "fjlapackex"
++    HINTS ${_SSL2_PATHS}
++    PATH_SUFFIXES "lib64"
++)
++find_path(
++    COSMA_SSL2_INCLUDE_DIRS
++    NAMES "cblas.h" 
++    HINTS ${_SSL2_PATHS}
++    PATH_SUFFIXES "include"
++)
++
++# check if found
++include(FindPackageHandleStandardArgs)
++find_package_handle_standard_args(SSL2 REQUIRED_VARS COSMA_SSL2_INCLUDE_DIRS COSMA_SSL2_LINK_LIBRARIES)
++
++# add target to link against
++if(NOT TARGET cosma::BLAS::SSL2::ssl2)
++  add_library(cosma::BLAS::SSL2::ssl2 INTERFACE IMPORTED)
++  add_library(cosma::BLAS::SSL2::blas ALIAS cosma::BLAS::SSL2::ssl2)
++endif()
++set_property(TARGET cosma::BLAS::SSL2::ssl2 PROPERTY INTERFACE_LINK_LIBRARIES ${COSMA_SSL2_LINK_LIBRARIES})
++set_property(TARGET cosma::BLAS::SSL2::ssl2 PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${COSMA_SSL2_INCLUDE_DIRS})
++
++# prevent clutter in cache
++MARK_AS_ADVANCED(SSL2_FOUND SSL2_LIBRARIES SSL2_INCLUDE_DIRS)
diff --git a/recipes/cp2k/2024.3/gh200/repo/packages/cosma/mpi-view.patch b/recipes/cp2k/2024.3/gh200/repo/packages/cosma/mpi-view.patch
@@ -0,0 +1,13 @@
+diff --git a/cmake/cosmaConfig.cmake.in b/cmake/cosmaConfig.cmake.in
+index cf7038f..1527c53 100644
+--- a/cmake/cosmaConfig.cmake.in
++++ b/cmake/cosmaConfig.cmake.in
+@@ -11,7 +11,7 @@ if(NOT TARGET cosma::cosma)
+     set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_MODULE_PATH})
+
+     set(MKL_ROOT "@MKL_ROOT@")
+-    set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
++    #    set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
+     set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
+     find_package(MPI COMPONENTS "CXX" REQUIRED)
+
diff --git a/recipes/cp2k/2024.3/gh200/repo/packages/cosma/package.py b/recipes/cp2k/2024.3/gh200/repo/packages/cosma/package.py
@@ -0,0 +1,135 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+
+
+class Cosma(CMakePackage):
+    """
+    Distributed Communication-Optimal Matrix-Matrix Multiplication Library
+    """
+
+    maintainers("haampie", "kabicm", "teonnik", "simonpintarelli", "mtaillefumier")
+    homepage = "https://github.com/eth-cscs/COSMA"
+    url = "https://github.com/eth-cscs/COSMA/archive/refs/tags/v2.6.6.tar.gz"
+    git = "https://github.com/eth-cscs/COSMA.git"
+
+    license("BSD-3-Clause")
+
+    # note: The default archives produced with github do not have the archives
+    #       of the submodules.
+    version("master", branch="master", submodules=False)
+    version("2.6.6", sha256="1604be101e77192fbcc5551236bc87888d336e402f5409bbdd9dea900401cc37")
+    version("2.6.5", sha256="10d9b7ecc1ce44ec5b9e0c0bf89278a63029912ec3ea99661be8576b553ececf")
+    version("2.6.4", sha256="6d7bd5e3005874af9542a329c93e7ccd29ca1a5573dae27618fac2704fa2b6ab")
+    version("2.6.3", sha256="c2a3735ea8f860930bea6706d968497d72a1be0498c689b5bc4a951ffc2d1146")
+    version("2.6.2", sha256="2debb5123cc35aeebc5fd2f8a46cfd6356d1e27618c9bb57129ecd09aa400940")
+    version("2.6.1", sha256="69aa6634a030674f0d9be61e7b0bf0dc17acf0fc9e7a90b40e3179e2254c8d67")
+    version("2.5.1", sha256="085b7787597374244bbb1eb89bc69bf58c35f6c85be805e881e1c0b25166c3ce")
+    version("2.5.0", sha256="7f68bb0ee5c80f9b8df858afcbd017ad4ed87ac09439d13d7d890844dbdd3d54")
+    version("2.4.0", sha256="5714315ce06d48037f86cfee2d7f19340643fee95e9d7f1e92dc1b623b67e395")
+    version("2.3.0", sha256="0c01c2deb5a0cd177952178350188a62c42ce55e604d7948ac472f55bf0d4815")
+    version("2.2.0", sha256="1eb92a98110df595070a12193b9221eecf9d103ced8836c960f6c79a2bd553ca")
+    version("2.0.7", sha256="8d70bfcbda6239b6a8fbeaca138790bbe58c0c3aa576879480d2632d4936cf7e")
+    version("2.0.2", sha256="4f3354828bc718f3eef2f0098c3bdca3499297497a220da32db1acd57920c68d")
+
+    # We just need the libraries of cuda and rocm, so no need to extend
+    # CudaPackage or ROCmPackage.
+    variant("cuda", default=False, description="Build with cuBLAS support")
+    variant("rocm", default=False, description="Build with rocBLAS support")
+    variant("scalapack", default=False, description="Build with ScaLAPACK API")
+    variant("shared", default=True, description="Build the shared library version")
+    variant("tests", default=False, description="Build tests")
+    variant("apps", default=False, description="Build miniapp")
+    variant("profiling", default=False, description="Enable profiling")
+    variant("gpu_direct", default=False, description="GPU aware MPI")
+
+    with when("+cuda"):
+        variant("nccl", default=False, description="Use cuda nccl")
+
+    with when("+rocm"):
+        variant("rccl", default=False, description="Use rocm rccl")
+
+    depends_on("[email protected]:", type="build")
+    depends_on("mpi@3:")
+    depends_on("blas", when="~cuda ~rocm")
+    depends_on("scalapack", when="+scalapack")
+    depends_on("cuda", when="+cuda")
+    depends_on("rocblas", when="+rocm")
+    depends_on("nccl", when="+nccl")
+    depends_on("rccl", when="+rccl")
+
+    with when("@2.6.3:"):
+        depends_on("[email protected]:+cuda", when="+cuda")
+        depends_on("[email protected]:+rocm", when="+rocm")
+
+    with when("@2.6.1:2.6.2"):
+        depends_on("[email protected]+rocm", when="+rocm")
+        depends_on("[email protected]+cuda", when="+cuda")
+
+    with when("@2.6.1:"):
+        depends_on("costa")
+        depends_on("costa+scalapack", when="+scalapack")
+        depends_on("cxxopts", when="+apps")
+        depends_on("cxxopts", when="+tests")
+        depends_on("semiprof", when="+profiling")
+        depends_on("costa+profiling", when="+profiling")
+
+    patch("fj-ssl2.patch", when="^fujitsu-ssl2")
+    patch("mpi-view.patch")
+
+    def setup_build_environment(self, env):
+        if "+cuda" in self.spec:
+            env.set("CUDA_PATH", self.spec["cuda"].prefix)
+
+    def cosma_blas_cmake_arg(self):
+        query_to_cmake_arg = [
+            ("+cuda", "CUDA"),
+            ("+rocm", "ROCM"),
+            ("^intel-mkl", "MKL"),
+            ("^intel-oneapi-mkl", "MKL"),
+            ("^cray-libsci", "CRAY_LIBSCI"),
+            ("^netlib-lapack", "CUSTOM"),
+            ("^openblas", "OPENBLAS"),
+            ("^fujitsu-ssl2", "SSL2"),
+        ]
+
+        if self.version >= Version("2.4.0"):
+            query_to_cmake_arg.extend(
+                [("^blis", "BLIS"), ("^amdblis", "BLIS"), ("^atlas", "ATLAS")]
+            )
+
+        for query, cmake_arg in query_to_cmake_arg:
+            if query in self.spec:
+                return cmake_arg
+
+        return "CUSTOM"
+
+    def cosma_scalapack_cmake_arg(self):
+        spec = self.spec
+
+        if "~scalapack" in spec:
+            return "OFF"
+        elif "^intel-mkl" in spec or "^intel-oneapi-mkl" in spec:
+            return "MKL"
+        elif "^cray-libsci" in spec:
+            return "CRAY_LIBSCI"
+
+        return "CUSTOM"
+
+    def cmake_args(self):
+        return [
+            self.define_from_variant("COSMA_WITH_TESTS", "tests"),
+            self.define_from_variant("COSMA_WITH_APPS", "apps"),
+            self.define_from_variant("COSMA_WITH_NCCL", "nccl"),
+            self.define_from_variant("COSMA_WITH_RCCL", "rccl"),
+            self.define_from_variant("COSMA_WITH_GPU_AWARE_MPI", "gpu_direct"),
+            self.define_from_variant("COSMA_WITH_PROFILING", "profiling"),
+            self.define("COSMA_WITH_BENCHMARKS", False),
+            self.define("COSMA_BLAS", self.cosma_blas_cmake_arg()),
+            self.define("COSMA_SCALAPACK", self.cosma_scalapack_cmake_arg()),
+            self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
+        ]
diff --git a/recipes/cp2k/2024.3/gh200/repo/packages/costa/mpi-view.patch b/recipes/cp2k/2024.3/gh200/repo/packages/costa/mpi-view.patch
@@ -0,0 +1,13 @@
+diff --git a/cmake/costaConfig.cmake.in b/cmake/costaConfig.cmake.in
+index 8fae534..d744c90 100644
+--- a/cmake/costaConfig.cmake.in
++++ b/cmake/costaConfig.cmake.in
+@@ -10,7 +10,7 @@ include(CMakeFindDependencyMacro)
+ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_MODULE_PATH})
+
+ set(MKL_ROOT "@MKL_ROOT@")
+-set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
++#set(MPI_CXX_COMPILER "@MPI_CXX_COMPILER@")
+
+ set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
+ set(COSTA_SCALAPACK "@COSTA_SCALAPACK@")