Lapack: cusolver TPL logic and support for gesv

CMakeLists.txt

@@ -375,6 +375,7 @@ ELSE()
   KOKKOSKERNELS_LINK_TPL(kokkoskernels PUBLIC MKL)
   KOKKOSKERNELS_LINK_TPL(kokkoskernels PUBLIC CUBLAS)
   KOKKOSKERNELS_LINK_TPL(kokkoskernels PUBLIC CUSPARSE)
+  KOKKOSKERNELS_LINK_TPL(kokkoskernels PUBLIC CUSOLVER)
   KOKKOSKERNELS_LINK_TPL(kokkoskernels PUBLIC ROCBLAS)
   KOKKOSKERNELS_LINK_TPL(kokkoskernels PUBLIC ROCSPARSE)
   KOKKOSKERNELS_LINK_TPL(kokkoskernels PUBLIC ROCSOLVER)

cm_generate_makefile.bash

@@ -178,6 +178,7 @@ get_kernels_tpls_list() {
   KOKKOSKERNELS_USER_TPL_LIBNAME_CMD=
   CUBLAS_DEFAULT=OFF
   CUSPARSE_DEFAULT=OFF
+  CUSOLVER_DEFAULT=OFF
   ROCBLAS_DEFAULT=OFF
   ROCSPARSE_DEFAULT=OFF
   PARSE_TPLS_LIST=$(echo $KOKKOSKERNELS_TPLS | tr "," "\n")
@@ -191,6 +192,9 @@ get_kernels_tpls_list() {
     if [ "$UC_TPLS" == "CUSPARSE" ]; then
       CUSPARSE_DEFAULT=ON
     fi
+    if [ "$UC_TPLS" == "CUSOLVER" ]; then
+      CUSOLVER_DEFAULT=ON
+    fi
     if [ "$UC_TPLS" == "ROCBLAS" ]; then
       ROCBLAS_DEFAULT=ON
     fi
@@ -224,6 +228,9 @@ get_kernels_tpls_list() {
   if [ "$CUSPARSE_DEFAULT" == "OFF" ]; then
     KOKKOSKERNELS_TPLS_CMD="-DKokkosKernels_ENABLE_TPL_CUSPARSE=OFF ${KOKKOSKERNELS_TPLS_CMD}"
   fi
+  if [ "$CUSOLVER_DEFAULT" == "OFF" ]; then
+    KOKKOSKERNELS_TPLS_CMD="-DKokkosKernels_ENABLE_TPL_CUSOLVER=OFF ${KOKKOSKERNELS_TPLS_CMD}"
+  fi
   if [ "$ROCBLAS_DEFAULT" == "OFF" ]; then
     KOKKOSKERNELS_TPLS_CMD="-DKokkosKernels_ENABLE_TPL_ROCBLAS=OFF ${KOKKOSKERNELS_TPLS_CMD}"
   fi

cmake/Dependencies.cmake

@@ -1,6 +1,6 @@
 TRIBITS_PACKAGE_DEFINE_DEPENDENCIES(
         LIB_REQUIRED_PACKAGES Kokkos
-        LIB_OPTIONAL_TPLS quadmath MKL BLAS LAPACK METIS SuperLU Cholmod CUBLAS CUSPARSE ROCBLAS ROCSPARSE
+        LIB_OPTIONAL_TPLS quadmath MKL BLAS LAPACK METIS SuperLU Cholmod CUBLAS CUSPARSE CUSOLVER ROCBLAS ROCSPARSE
         TEST_OPTIONAL_TPLS yaml-cpp
 )
 # NOTE: If you update names in LIB_OPTIONAL_TPLS above, make sure to map those names in

cmake/KokkosKernels_config.h.in

@@ -114,10 +114,12 @@
 #cmakedefine KOKKOSKERNELS_ENABLE_TPL_LAPACK
 /* MKL library */
 #cmakedefine KOKKOSKERNELS_ENABLE_TPL_MKL
-/* CUSPARSE */
-#cmakedefine KOKKOSKERNELS_ENABLE_TPL_CUSPARSE
 /* CUBLAS */
 #cmakedefine KOKKOSKERNELS_ENABLE_TPL_CUBLAS
+/* CUSPARSE */
+#cmakedefine KOKKOSKERNELS_ENABLE_TPL_CUSPARSE
+/* CUSOLVER */
+#cmakedefine KOKKOSKERNELS_ENABLE_TPL_CUSOLVER
 /* MAGMA */
 #cmakedefine KOKKOSKERNELS_ENABLE_TPL_MAGMA
 /* SuperLU */

cmake/Modules/FindTPLCUSOLVER.cmake

@@ -0,0 +1,17 @@
+FIND_PACKAGE(CUDA)
+
+INCLUDE(FindPackageHandleStandardArgs)
+IF (NOT CUDA_FOUND)
+  #Important note here: this find Module is named TPLCUSOLVER
+  #The eventual target is named CUSOLVER. To avoid naming conflicts
+  #the find module is called TPLCUSOLVER. This call will cause
+  #the find_package call to fail in a "standard" CMake way
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(TPLCUSOLVER REQUIRED_VARS CUDA_FOUND)
+ELSE()
+  #The libraries might be empty - OR they might explicitly be not found
+  IF("${CUDA_cusolver_LIBRARY}" MATCHES "NOTFOUND")
+    FIND_PACKAGE_HANDLE_STANDARD_ARGS(TPLCUSOLVER REQUIRED_VARS CUDA_cusolver_LIBRARY)
+  ELSE()
+     KOKKOSKERNELS_CREATE_IMPORTED_TPL(CUSOLVER LIBRARY ${CUDA_cusolver_LIBRARY})
+  ENDIF()
+ENDIF()

cmake/kokkoskernels_features.cmake

@@ -39,3 +39,11 @@ ELSEIF (KOKKOSKERNELS_ENABLE_TPL_ROCSOLVER AND NOT KOKKOSKERNELS_ENABLE_TPL_ROCS
 ELSEIF (KOKKOSKERNELS_ENABLE_TPL_ROCSOLVER AND NOT KOKKOSKERNELS_ENABLE_TPL_ROCBLAS)
   MESSAGE(FATAL_ERROR "rocSOLVER requires rocBLAS, please reconfigure with KOKKOSKERNELS_ENABLE_TPL_ROCBLAS:BOOL=ON.")
 ENDIF()
+
+IF (KOKKOSKERNELS_ENABLE_TPL_CUSOLVER AND NOT KOKKOSKERNELS_ENABLE_TPL_CUBLAS AND NOT KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
+  MESSAGE(FATAL_ERROR "cuSOLVER requires cuBLAS and cuSPARSE, please reconfigure with KOKKOSKERNELS_ENABLE_TPL_CUBLAS:BOOL=ON and KOKKOSKERNELS_ENABLE_TPL_CUSPARSE:BOOL=ON.")
+ELSEIF (KOKKOSKERNELS_ENABLE_TPL_CUSOLVER AND NOT KOKKOSKERNELS_ENABLE_TPL_CUSPARSE)
+  MESSAGE(FATAL_ERROR "cuSOLVER requires cuSPARSE, please reconfigure with KOKKOSKERNELS_ENABLE_TPL_CUSPARSE:BOOL=ON.")
+ELSEIF (KOKKOSKERNELS_ENABLE_TPL_CUSOLVER AND NOT KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
+  MESSAGE(FATAL_ERROR "cuSOLVER requires cuBLAS, please reconfigure with KOKKOSKERNELS_ENABLE_TPL_CUBLAS:BOOL=ON.")
+ENDIF()

cmake/kokkoskernels_tpls.cmake

@@ -447,14 +447,18 @@ ENDIF()
 KOKKOSKERNELS_ADD_OPTION(NO_DEFAULT_CUDA_TPLS OFF BOOL "Whether CUDA TPLs should be enabled by default. Default: OFF")
 SET(CUBLAS_DEFAULT   ${KOKKOS_ENABLE_CUDA})
 SET(CUSPARSE_DEFAULT ${KOKKOS_ENABLE_CUDA})
+SET(CUSOLVER_DEFAULT ${KOKKOS_ENABLE_CUDA})
 IF(KOKKOSKERNELS_NO_DEFAULT_CUDA_TPLS)
   SET(CUBLAS_DEFAULT   OFF)
   SET(CUSPARSE_DEFAULT OFF)
+  SET(CUSOLVER_DEFAULT OFF)
 ENDIF()
 KOKKOSKERNELS_ADD_TPL_OPTION(CUBLAS   ${CUBLAS_DEFAULT}     "Whether to enable CUBLAS"
   DEFAULT_DOCSTRING "ON if CUDA-enabled Kokkos, otherwise OFF")
 KOKKOSKERNELS_ADD_TPL_OPTION(CUSPARSE ${CUSPARSE_DEFAULT}   "Whether to enable CUSPARSE"
   DEFAULT_DOCSTRING "ON if CUDA-enabled Kokkos, otherwise OFF")
+KOKKOSKERNELS_ADD_TPL_OPTION(CUSOLVER ${CUSOLVER_DEFAULT}   "Whether to enable CUSOLVER"
+  DEFAULT_DOCSTRING "ON if CUDA-enabled Kokkos, otherwise OFF")
 
 KOKKOSKERNELS_ADD_OPTION(NO_DEFAULT_ROCM_TPLS OFF BOOL "Whether ROCM TPLs should be enabled by default. Default: OFF")
 # Unlike CUDA, ROCm does not automatically install these TPLs
@@ -501,6 +505,7 @@ IF (NOT KOKKOSKERNELS_HAS_TRILINOS)
   KOKKOSKERNELS_IMPORT_TPL(MKL)
   KOKKOSKERNELS_IMPORT_TPL(CUBLAS)
   KOKKOSKERNELS_IMPORT_TPL(CUSPARSE)
+  KOKKOSKERNELS_IMPORT_TPL(CUSOLVER)
   KOKKOSKERNELS_IMPORT_TPL(CBLAS)
   KOKKOSKERNELS_IMPORT_TPL(LAPACKE)
   KOKKOSKERNELS_IMPORT_TPL(CHOLMOD)

common/src/KokkosKernels_PrintConfiguration.hpp

@@ -44,6 +44,18 @@ inline void print_cusparse_version_if_enabled(std::ostream& os) {
      << "KOKKOSKERNELS_ENABLE_TPL_CUSPARSE: no\n";
 #endif
 }
+
+inline void print_cusolver_version_if_enabled(std::ostream& os) {
+#ifdef KOKKOSKERNELS_ENABLE_TPL_CUSOLVER
+  os << "  "
+     << "KOKKOSKERNELS_ENABLE_TPL_CUSOLVER: " << cusolver_version_string()
+     << "\n";
+#else
+  os << "  "
+     << "KOKKOSKERNELS_ENABLE_TPL_CUSOLVER: no\n";
+#endif
+}
+
 inline void print_enabled_tpls(std::ostream& os) {
 #ifdef KOKKOSKERNELS_ENABLE_TPL_LAPACK
   os << "  "

common/src/KokkosKernels_TplsVersion.hpp

@@ -28,6 +28,10 @@
 #include "cusparse.h"
 #endif
 
+#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSOLVER)
+#include "cusolver_common.h"
+#endif
+
 namespace KokkosKernels {
 
 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
@@ -53,5 +57,16 @@ inline std::string cusparse_version_string() {
 }
 #endif
 
+#if defined(KOKKOSKERNELS_ENABLE_TPL_CUSOLVER)
+inline std::string cusolver_version_string() {
+  std::stringstream ss;
+
+  ss << CUSOLVER_VER_MAJOR << "." << CUSOLVER_VER_MINOR << "."
+     << CUSOLVER_VER_PATCH << "." << CUSOLVER_VER_BUILD;
+
+  return ss.str();
+}
+#endif
+
 }  // namespace KokkosKernels
 #endif  // _KOKKOSKERNELS_TPLS_VERSIONS_HPP

lapack/impl/KokkosLapack_gesv_spec.hpp

@@ -90,7 +90,7 @@ struct GESV<ExecutionSpace, AMatrix, BXMV, IPIVV, false,
     // NOTE: Might add the implementation of KokkosLapack::gesv later
     throw std::runtime_error(
         "No fallback implementation of GESV (general LU factorization & solve) "
-        "exists. Enable LAPACK and/or MAGMA TPL.");
+        "exists. Enable LAPACK, CUSOLVER, ROCSOLVER or MAGMA TPL.");
   }
 };
 

lapack/tpls/KokkosLapack_Cuda_tpl.hpp

@@ -17,7 +17,7 @@
 #define KOKKOSLAPACK_CUDA_TPL_HPP_
 
 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSOLVER)
-#include <KokkosLapack_tpl_spec.hpp>
+#include "KokkosLapack_cusolver.hpp"
 
 namespace KokkosLapack {
 namespace Impl {

lapack/tpls/KokkosLapack_cusolver.hpp

@@ -0,0 +1,92 @@
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
+// See https://kokkos.org/LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//@HEADER
+
+#ifndef KOKKOSLAPACK_CUSOLVER_HPP_
+#define KOKKOSLAPACK_CUSOLVER_HPP_
+
+#ifdef KOKKOSKERNELS_ENABLE_TPL_CUSOLVER
+#include <cusolverDn.h>
+
+namespace KokkosLapack {
+namespace Impl {
+
+// Declaration of the singleton for cusolver
+// this is the only header that needs to be
+// included when using cusolverDn.
+struct CudaLapackSingleton {
+  cusolverDnHandle_t handle;
+
+  CudaLapackSingleton();
+
+  static CudaLapackSingleton& singleton();
+};
+
+inline void cusolver_internal_error_throw(cusolverStatus_t cusolverStatus,
+                                          const char* name, const char* file,
+                                          const int line) {
+  std::ostringstream out;
+  out << name << " error( ";
+  switch (cusolverStatus) {
+    case CUSOLVER_STATUS_NOT_INITIALIZED:
+      out << "CUSOLVER_STATUS_NOT_INITIALIZED): cusolver handle was not "
+             "created correctly.";
+      break;
+    case CUSOLVER_STATUS_ALLOC_FAILED:
+      out << "CUSOLVER_STATUS_ALLOC_FAILED): you might tried to allocate too "
+             "much memory";
+      break;
+    case CUSOLVER_STATUS_INVALID_VALUE:
+      out << "CUSOLVER_STATUS_INVALID_VALUE)";
+      break;
+    case CUSOLVER_STATUS_ARCH_MISMATCH:
+      out << "CUSOLVER_STATUS_ARCH_MISMATCH)";
+      break;
+    case CUSOLVER_STATUS_EXECUTION_FAILED:
+      out << "CUSOLVER_STATUS_EXECUTION_FAILED)";
+      break;
+    case CUSOLVER_STATUS_INTERNAL_ERROR:
+      out << "CUSOLVER_STATUS_INTERNAL_ERROR)";
+      break;
+    case CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED:
+      out << "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED)";
+      break;
+    default: out << "unrecognized error code): this is bad!"; break;
+  }
+  if (file) {
+    out << " " << file << ":" << line;
+  }
+  throw std::runtime_error(out.str());
+}
+
+inline void cusolver_internal_safe_call(cusolverStatus_t cusolverStatus,
+                                        const char* name,
+                                        const char* file = nullptr,
+                                        const int line   = 0) {
+  if (CUSOLVER_STATUS_SUCCESS != cusolverStatus) {
+    cusolver_internal_error_throw(cusolverStatus, name, file, line);
+  }
+}
+
+// The macro below defines is the public interface for the safe cusolver calls.
+// The functions themselves are protected by impl namespace.
+#define KOKKOS_CUSOLVER_SAFE_CALL_IMPL(call)                             \
+  KokkosLapack::Impl::cusolver_internal_safe_call(call, #call, __FILE__, \
+                                                  __LINE__)
+
+}  // namespace Impl
+}  // namespace KokkosLapack
+#endif  // KOKKOSKERNELS_ENABLE_TPL_CUSOLVER
+#endif  // KOKKOSLAPACK_CUSOLVER_HPP_

lapack/tpls/KokkosLapack_gesv_tpl_spec_avail.hpp

@@ -79,6 +79,37 @@ KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_MAGMA(Kokkos::complex<float>,
 }  // namespace Impl
 }  // namespace KokkosLapack
 
+// CUSOLVER
+#ifdef KOKKOSKERNELS_ENABLE_TPL_CUSOLVER
+namespace KokkosLapack {
+namespace Impl {
+
+#define KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_CUSOLVER(SCALAR, LAYOUT, MEMSPACE)  \
+  template <>                                                                \
+  struct gesv_tpl_spec_avail<                                                \
+      Kokkos::Cuda,                                                          \
+      Kokkos::View<SCALAR**, LAYOUT, Kokkos::Device<Kokkos::Cuda, MEMSPACE>, \
+                   Kokkos::MemoryTraits<Kokkos::Unmanaged> >,                \
+      Kokkos::View<SCALAR**, LAYOUT, Kokkos::Device<Kokkos::Cuda, MEMSPACE>, \
+                   Kokkos::MemoryTraits<Kokkos::Unmanaged> >,                \
+      Kokkos::View<int*, LAYOUT, Kokkos::Device<Kokkos::Cuda, MEMSPACE>,     \
+                   Kokkos::MemoryTraits<Kokkos::Unmanaged> > > {             \
+    enum : bool { value = true };                                            \
+  };
+
+KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_CUSOLVER(double, Kokkos::LayoutLeft,
+                                          Kokkos::CudaSpace)
+KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_CUSOLVER(float, Kokkos::LayoutLeft,
+                                          Kokkos::CudaSpace)
+KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_CUSOLVER(Kokkos::complex<double>,
+                                          Kokkos::LayoutLeft, Kokkos::CudaSpace)
+KOKKOSLAPACK_GESV_TPL_SPEC_AVAIL_CUSOLVER(Kokkos::complex<float>,
+                                          Kokkos::LayoutLeft, Kokkos::CudaSpace)
+
+}  // namespace Impl
+}  // namespace KokkosLapack
+#endif  // CUSOLVER
+
 #ifdef KOKKOSKERNELS_ENABLE_TPL_ROCSOLVER
 #include <rocsolver/rocsolver.h>