diff --git a/ci/common.yml b/ci/common.yml index 8a9a1146..0e0aaceb 100644 --- a/ci/common.yml +++ b/ci/common.yml @@ -6,8 +6,9 @@ stages: stage: build variables: SLURM_TIMELIMIT: 180 + SPACK_DEVELOP: "" script: - - ./stack-build -n $STACK_NAME -s $STACK_SYSTEM -r $STACK_RECIPE -b /dev/shm/jenkssl + - ./stack-build -n $STACK_NAME -s $STACK_SYSTEM -r $STACK_RECIPE -b /dev/shm/jenkssl $SPACK_DEVELOP after_script: - rm -Rf /dev/shm/jenkssl artifacts: diff --git a/docs/uenv-cp2k.md b/docs/uenv-cp2k.md new file mode 100644 index 00000000..10bac81b --- /dev/null +++ b/docs/uenv-cp2k.md @@ -0,0 +1,39 @@ +# CP2K + +[CP2K](https://www.cp2k.org/) version `2023.2`. + +An environment that provides the latest version of [CP2K](https://www.cp2k.org/), along with the libraries and tools required to build a different or custom version of [CP2K](https://www.cp2k.org/). + +The following environment views are provided: + +* `cp2k-scalapack`: CP2K, dependencies, and [ScaLAPACK](https://www.netlib.org/scalapack/) as diagonalization library +* `cp2k-scalapack-dev`: dependencies and [ScaLAPACK](https://www.netlib.org/scalapack/) +* `cp2k-elpa`: CP2K, dependencies, and [ELPA](https://elpa.mpcdf.mpg.de/) as diagonalization library +* `cp2k-elpa-dev`: dependencies and [ELPA](https://elpa.mpcdf.mpg.de/) + +## Building a custom version of CP2K + +### Using modules + +To build your version of CP2K do the following steps: + +```bash +# Load the required modules +module load [...] +cd cp2k +mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Release -DCP2K_SCALAPACK_VENDOR=MKL -DCP2K_USE_ACCEL=cuda -DCP2K_WITH_GPU=A100 +make -j20 +``` + +See CP2K's [README_cmake.md](https://github.com/cp2k/cp2k/blob/master/README_cmake.md) for details. + +### Using Spack + +```bash +uenv start cp2k-a100.squashfs +export SPACK_SYSTEM_CONFIG_PATH=/user-environment/config/ +spack install cp2k [...] +``` + +See Spack's [CP2K package](https://packages.spack.io/package.html?name=cp2k) for details. diff --git a/mkdocs.yml b/mkdocs.yml index 328822c7..cdde94f6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,6 +4,7 @@ nav: - 'Environments': - 'gromacs': uenv-gromacs.md - 'quantum-espresso': uenv-qe.md + - 'cp2k': uenv-cp2k.md theme: name: material features: diff --git a/recipes/cp2k/a100/compilers.yaml b/recipes/cp2k/a100/compilers.yaml new file mode 100644 index 00000000..5db9c480 --- /dev/null +++ b/recipes/cp2k/a100/compilers.yaml @@ -0,0 +1,5 @@ +bootstrap: + spec: gcc@11 +gcc: + specs: + - gcc@11 diff --git a/recipes/cp2k/a100/config.yaml b/recipes/cp2k/a100/config.yaml new file mode 100644 index 00000000..0a259cab --- /dev/null +++ b/recipes/cp2k/a100/config.yaml @@ -0,0 +1,6 @@ +name: cp2k-nvgpu +store: /user-environment +spack: + repo: https://github.com/spack/spack.git + commit: develop +modules: True diff --git a/recipes/cp2k/a100/environments.yaml b/recipes/cp2k/a100/environments.yaml new file mode 100644 index 00000000..fd31c86d --- /dev/null +++ b/recipes/cp2k/a100/environments.yaml @@ -0,0 +1,67 @@ +cp2k-nvgpu: + compiler: + - toolchain: gcc + spec: gcc@11 + unify: true + specs: + - cuda@11.8 + - intel-oneapi-mkl +cluster + - fftw@3.3.10 +openmp + - libxc ~cuda + - sirius +fortran +scalapack + - dbcsr@2.6.0 +cuda +shared +openmp ^intel-oneapi-mkl ^cray-mpich + - cosma@master +gpu_direct +scalapack +shared ^intel-oneapi-mkl ^cray-mpich + - cp2k@2023.2 +libxc +libint +spglib +cosma +sirius ~elpa build_system=cmake ^intel-oneapi-mkl ^cray-mpich + variants: + - +mpi + - +cuda + - cuda_arch=80 + - build_type=Release + views: + cp2k-scalapack-dev: + exclude: ["cp2k"] + cp2k-scalapack: + exclude: [""] + mpi: + spec: cray-mpich + gpu: cuda +cp2k-nvgpu-elpa: + compiler: + - toolchain: gcc + spec: gcc@11 + unify: true + specs: + - cuda@11.8 + - intel-oneapi-mkl +cluster + - fftw@3.3.10 +openmp + - libxc ~cuda + - sirius +fortran + scalapack +elpa + - dbcsr@2.6.0 +cuda +shared +openmp ^intel-oneapi-mkl ^cray-mpich + - cosma@master +gpu_direct +scalapack +shared ^intel-oneapi-mkl ^cray-mpich + - cp2k@2023.2 +libxc +libint +spglib +cosma +sirius +elpa build_system=cmake ^intel-oneapi-mkl ^cray-mpich + variants: + - +mpi + - +cuda + - cuda_arch=80 + - build_type=Release + views: + cp2k-elpa-dev: + exclude: ["cp2k"] + cp2k-elpa: + exclude: [""] + mpi: + spec: cray-mpich + gpu: cuda +tools: + compiler: + - toolchain: gcc + spec: gcc@11 + unify: true + specs: + - cmake@3.25.2 + - pkgconfig + - python@3.10 + - py-numpy + views: + tools: + exclude: [""] diff --git a/recipes/cp2k/a100/modules.yaml b/recipes/cp2k/a100/modules.yaml new file mode 100644 index 00000000..61c5d4c1 --- /dev/null +++ b/recipes/cp2k/a100/modules.yaml @@ -0,0 +1,25 @@ +modules: + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + projections: + all: '{name}/{version}' + cp2k~elpa: 'cp2k/{version}' + sirius~elpa: 'sirius/{version}' + cp2k+elpa: 'cp2k+elpa/{version}' + sirius+elpa: 'sirius+elpa/{version}' diff --git a/recipes/cp2k/a100/repo/packages/intel-oneapi-mkl/README.md b/recipes/cp2k/a100/repo/packages/intel-oneapi-mkl/README.md new file mode 100644 index 00000000..7c47db61 --- /dev/null +++ b/recipes/cp2k/a100/repo/packages/intel-oneapi-mkl/README.md @@ -0,0 +1,3 @@ +# `intel-oneapi-mkl` + +Custom version of the `intel-oneapi-mkl` library that does not provide `fftw-api@3`. diff --git a/recipes/cp2k/a100/repo/packages/intel-oneapi-mkl/package.py b/recipes/cp2k/a100/repo/packages/intel-oneapi-mkl/package.py new file mode 100644 index 00000000..a488bca2 --- /dev/null +++ b/recipes/cp2k/a100/repo/packages/intel-oneapi-mkl/package.py @@ -0,0 +1,212 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from os.path import dirname, isdir + +from spack.package import * + + +@IntelOneApiPackage.update_description +class IntelOneapiMkl(IntelOneApiLibraryPackage): + """Intel oneAPI Math Kernel Library (Intel oneMKL; formerly Intel Math + Kernel Library or Intel MKL), is a library of optimized math + routines for science, engineering, and financial + applications. Core math functions include BLAS, LAPACK, + ScaLAPACK, sparse solvers, fast Fourier transforms, and vector + math. + + """ + + maintainers("rscohn2") + + homepage = ( + "https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html" + ) + + version( + "2023.1.0", + url="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/cd17b7fe-500e-4305-a89b-bd5b42bfd9f8/l_onemkl_p_2023.1.0.46342_offline.sh", + sha256="cc28c94cab23c185520b93c5a04f3979d8da6b4c90cee8c0681dd89819d76167", + expand=False, + ) + version( + "2023.0.0", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/19138/l_onemkl_p_2023.0.0.25398_offline.sh", + sha256="0d61188e91a57bdb575782eb47a05ae99ea8eebefee6b2dfe20c6708e16e9927", + expand=False, + ) + version( + "2022.2.1", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/19038/l_onemkl_p_2022.2.1.16993_offline.sh", + sha256="eedd4b795720de776b1fc5f542ae0fac37ec235cdb567f7c2ee3182e73e3e59d", + expand=False, + ) + version( + "2022.2.0", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/18898/l_onemkl_p_2022.2.0.8748_offline.sh", + sha256="07d7caedd4b9f025c6fd439a0d2c2f279b18ecbbb63cadb864f6c63c1ed942db", + expand=False, + ) + version( + "2022.1.0", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/18721/l_onemkl_p_2022.1.0.223_offline.sh", + sha256="4b325a3c4c56e52f4ce6c8fbb55d7684adc16425000afc860464c0f29ea4563e", + expand=False, + ) + version( + "2022.0.2", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/18483/l_onemkl_p_2022.0.2.136_offline.sh", + sha256="134b748825a474acc862bb4a7fada99741a15b7627cfaa6ba0fb05ec0b902b5e", + expand=False, + ) + version( + "2022.0.1", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/18444/l_onemkl_p_2022.0.1.117_offline.sh", + sha256="22afafbe2f3762eca052ac21ec40b845ff2f3646077295c88c2f37f80a0cc160", + expand=False, + ) + version( + "2021.4.0", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/18222/l_onemkl_p_2021.4.0.640_offline.sh", + sha256="9ad546f05a421b4f439e8557fd0f2d83d5e299b0d9bd84bdd86be6feba0c3915", + expand=False, + ) + version( + "2021.3.0", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/17901/l_onemkl_p_2021.3.0.520_offline.sh", + sha256="a06e1cdbfd8becc63440b473b153659885f25a6e3c4dcb2907ad9cd0c3ad59ce", + expand=False, + ) + version( + "2021.2.0", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/17757/l_onemkl_p_2021.2.0.296_offline.sh", + sha256="816e9df26ff331d6c0751b86ed5f7d243f9f172e76f14e83b32bf4d1d619dbae", + expand=False, + ) + version( + "2021.1.1", + url="https://registrationcenter-download.intel.com/akdlm/irc_nas/17402/l_onemkl_p_2021.1.1.52_offline.sh", + sha256="818b6bd9a6c116f4578cda3151da0612ec9c3ce8b2c8a64730d625ce5b13cc0c", + expand=False, + ) + + variant("shared", default=True, description="Builds shared library") + variant("ilp64", default=False, description="Build with ILP64 support") + variant( + "cluster", default=False, description="Build with cluster support: scalapack, blacs, etc" + ) + variant( + "threads", + default="none", + description="Multithreading support", + values=("openmp", "tbb", "none"), + multi=False, + ) + + depends_on("tbb") + # cluster libraries need mpi + depends_on("mpi", when="+cluster") + + #provides("fftw-api@3") + provides("scalapack", when="+cluster") + provides("mkl") + provides("lapack") + provides("blas") + + @property + def component_dir(self): + return "mkl" + + @property + def headers(self): + return find_headers("*", self.component_prefix.include) + + @property + def libs(self): + shared = self.spec.satisfies("+shared") + + libs = self._find_mkl_libs(shared) + + system_libs = find_system_libraries(["libpthread", "libm", "libdl"]) + if shared: + return libs + system_libs + else: + return IntelOneApiStaticLibraryList(libs, system_libs) + + def setup_run_environment(self, env): + super().setup_run_environment(env) + + # Support RPATH injection to the library directories when the '-mkl' or '-qmkl' + # flag of the Intel compilers are used outside the Spack build environment. We + # should not try to take care of other compilers because the users have to + # provide the linker flags anyway and are expected to take care of the RPATHs + # flags too. We prefer the __INTEL_POST_CFLAGS/__INTEL_POST_FFLAGS flags over + # the PRE ones so that any other RPATHs provided by the users on the command + # line come before and take precedence over the ones we inject here. + for d in self._find_mkl_libs(self.spec.satisfies("+shared")).directories: + flag = "-Wl,-rpath,{0}".format(d) + env.append_path("__INTEL_POST_CFLAGS", flag, separator=" ") + env.append_path("__INTEL_POST_FFLAGS", flag, separator=" ") + + def setup_dependent_build_environment(self, env, dependent_spec): + # Only if environment modifications are desired (default is +envmods) + if self.spec.satisfies("+envmods"): + env.set("MKLROOT", self.component_prefix) + env.append_path("PKG_CONFIG_PATH", self.component_prefix.lib.pkgconfig) + + def _find_mkl_libs(self, shared): + libs = [] + + if self.spec.satisfies("+cluster"): + libs.extend([self._xlp64_lib("libmkl_scalapack"), "libmkl_cdft_core"]) + + libs.append(self._xlp64_lib("libmkl_intel")) + if self.spec.satisfies("threads=tbb"): + libs.append("libmkl_tbb_thread") + elif self.spec.satisfies("threads=openmp"): + if self.spec.satisfies("%oneapi") or self.spec.satisfies("%intel"): + libs.append("libmkl_intel_thread") + else: + libs.append("libmkl_gnu_thread") + else: + libs.append("libmkl_sequential") + + libs.append("libmkl_core") + + if self.spec.satisfies("+cluster"): + if any( + self.spec.satisfies(m) + for m in ["^intel-oneapi-mpi", "^intel-mpi", "^mpich", "^cray-mpich"] + ): + libs.append(self._xlp64_lib("libmkl_blacs_intelmpi")) + elif self.spec.satisfies("^openmpi"): + libs.append(self._xlp64_lib("libmkl_blacs_openmpi")) + else: + raise RuntimeError( + ( + "intel-oneapi-mpi +cluster requires one of " + "^intel-oneapi-mpi, ^intel-mpi, ^mpich, or ^openmpi" + ) + ) + + lib_path = self.component_prefix.lib.intel64 + lib_path = lib_path if isdir(lib_path) else dirname(lib_path) + + resolved_libs = find_libraries(libs, lib_path, shared=shared) + # Add MPI libraries for cluster support. If MPI is not in the + # spec, then MKL is externally installed and application must + # link with MPI libaries + if self.spec.satisfies("+cluster ^mpi"): + resolved_libs = resolved_libs + self.spec["mpi"].libs + return resolved_libs + + def _xlp64_lib(self, lib): + return lib + ("_ilp64" if self.spec.satisfies("+ilp64") else "_lp64") + + @run_after("install") + def fixup_prefix(self): + self.symlink_dir(self.component_prefix.include, self.prefix.include) + self.symlink_dir(self.component_prefix.lib, self.prefix.lib) + diff --git a/recipes/cp2k/ci.yml b/recipes/cp2k/ci.yml new file mode 100644 index 00000000..3eac4c75 --- /dev/null +++ b/recipes/cp2k/ci.yml @@ -0,0 +1,8 @@ +include: '/ci/common.yml' + +build-a100: + extends: .stack-build-hohgant-a100 + variables: + STACK_NAME: cp2k-a100 + STACK_RECIPE: recipes/cp2k/a100 + SPACK_DEVELOP: "-d"