Move Comm into a file

Add force module that uses KRS

src/CMakeLists.txt

@@ -20,6 +20,8 @@ if (ENABLE_KOKKOS_REMOTE_SPACES)
   target_compile_definitions(ExaMiniMD PRIVATE EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
   target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_SCALAR)
   #target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_HALO)
+  #target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_HALO_LOCAL)
+  #target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_LOCAL_GLOBAL)
   target_compile_definitions(ExaMiniMD PRIVATE SHMEMTESTS_USE_GLOBAL)
 endif()
 

src/comm_lib.cpp

@@ -0,0 +1,88 @@
+//************************************************************************
+//  ExaMiniMD v. 1.0
+//  Copyright (2018) National Technology & Engineering Solutions of Sandia,
+//  LLC (NTESS).
+//
+//  Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
+//  retains certain rights in this software.
+//
+//  ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
+//  use in source and binary forms, with or without modification, are
+//  permitted provided that the following conditions are met:
+//
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//
+//    2. Redistributions in binary form must reproduce the above copyright notice,
+//       this list of conditions and the following disclaimer in the documentation
+//       and/or other materials provided with the distribution.
+//
+//    3. Neither the name of the Corporation nor the names of the contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
+//  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+//  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+//  IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+//  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+//  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+//  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+//  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+//  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  POSSIBILITY OF SUCH DAMAGE.
+//
+//  Questions? Contact Christian R. Trott ([email protected])
+//************************************************************************
+
+#include <comm_lib.h>
+#include <assert.h>
+
+#if EXAMINIMD_ENABLE_MPI
+#include <mpi.h>
+#endif
+
+#ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
+#include <Kokkos_RemoteSpaces.hpp>
+#endif
+
+void comm_lib_init(int argc, char* argv[]) {
+#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
+  int mpi_thread_level_available;
+  int mpi_thread_level_required = MPI_THREAD_MULTIPLE;
+
+#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_SERIAL
+  mpi_thread_level_required = MPI_THREAD_SINGLE;
+#endif
+
+  MPI_Init_thread(&argc, &argv, mpi_thread_level_required,
+                  &mpi_thread_level_available);
+  assert(mpi_thread_level_available >= mpi_thread_level_required);
+
+#ifdef KRS_ENABLE_SHMEMSPACE
+  shmem_init_thread(mpi_thread_level_required, &mpi_thread_level_available);
+  assert(mpi_thread_level_available >= mpi_thread_level_required);
+#endif
+
+#ifdef KRS_ENABLE_NVSHMEMSPACE
+  MPI_Comm mpi_comm;
+  nvshmemx_init_attr_t attr;
+  mpi_comm      = MPI_COMM_WORLD;
+  attr.mpi_comm = &mpi_comm;
+  nvshmemx_init_attr(NVSHMEMX_INIT_WITH_MPI_COMM, &attr);
+#endif
+}
+
+void comm_lib_finalize() {
+#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
+#ifdef KRS_ENABLE_SHMEMSPACE
+  shmem_finalize();
+#endif
+#ifdef KRS_ENABLE_NVSHMEMSPACE
+  nvshmem_finalize();
+#endif
+  MPI_Finalize();
+#endif
+#endif
+}

src/comm_lib.h

@@ -0,0 +1,47 @@
+//************************************************************************
+//  ExaMiniMD v. 1.0
+//  Copyright (2018) National Technology & Engineering Solutions of Sandia,
+//  LLC (NTESS).
+//
+//  Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
+//  retains certain rights in this software.
+//
+//  ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
+//  use in source and binary forms, with or without modification, are
+//  permitted provided that the following conditions are met:
+//
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//
+//    2. Redistributions in binary form must reproduce the above copyright notice,
+//       this list of conditions and the following disclaimer in the documentation
+//       and/or other materials provided with the distribution.
+//
+//    3. Neither the name of the Corporation nor the names of the contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
+//  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+//  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+//  IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+//  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+//  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+//  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+//  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+//  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  POSSIBILITY OF SUCH DAMAGE.
+//
+//  Questions? Contact Christian R. Trott ([email protected])
+//************************************************************************
+
+#pragma once
+
+#ifndef COMM_INIT_H
+#define COMM_INIT_H
+
+void comm_lib_init(int argc, char* argv[]);
+void comm_lib_finalize();
+
+#endif

src/comm_types/CMakeLists.txt

@@ -1,9 +1,3 @@
 FILE(GLOB SRCS *.cpp)
 target_sources(ExaMiniMD PRIVATE ${SRCS})
-
-if (!ENABLE_MPI AND !ENABLE_KOKKOS_REMOTE_SPACES) 
-    # Skip MPI module
-    list(FILTER SRCS EXCLUDE REGEX ".*comm_mpi\\.cpp$")
-endif()
-
 target_sources(ExaMiniMD PRIVATE ${SRCS})

src/comm_types/comm_mpi.cpp

@@ -36,6 +36,8 @@
 //  Questions? Contact Christian R. Trott ([email protected])
 //************************************************************************
 
+#include <string>
+
 #if defined(EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
 #include<comm_mpi.h>
 
@@ -390,7 +392,8 @@ void CommMPI::exchange_halo() {
 };
 
 void CommMPI::update_halo() {
-#ifndef SHMEMTESTS_USE_HALO
+
+#if !defined(SHMEMTESTS_USE_HALO) && defined(EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
   return;
 #else
   Kokkos::Profiling::pushRegion("Comm::update_halo");
@@ -478,7 +481,13 @@ void CommMPI::update_force() {
   Kokkos::Profiling::popRegion();
 };
 
-const char* CommMPI::name() { return "CommMPI"; }
+const char* CommMPI::name() {
+    comm_name = std::string("CommMPI");
+    #ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
+    comm_name += "Distrib";
+    #endif
+    return comm_name.c_str();
+}
 
 int CommMPI::process_rank() { return proc_rank; }
 int CommMPI::num_processes() { return proc_size; }

src/comm_types/comm_mpi.h

@@ -70,7 +70,6 @@ class CommMPI: public Comm {
   System s;
 
   // Owned Variables
-
   int phase; // Communication Phase
   int proc_neighbors_recv[6]; // Neighbor for each phase
   int proc_neighbors_send[6]; // Neighbor for each phase
@@ -81,6 +80,8 @@ class CommMPI: public Comm {
   int proc_rank;         // My Process rank
   int proc_size;         // Number of processes
 
+  std::string comm_name;
+
   T_INT num_ghost[6];
   T_INT ghost_offsets[6];
 
@@ -503,7 +504,9 @@ class CommMPI: public Comm {
   KOKKOS_INLINE_FUNCTION
   void operator() (const TagCreateGlobalIndecies,
                    const T_INT& i) const {
+    #ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
     s.global_index(i) = N_MAX_MASK * proc_rank + i; 
+    #endif
   }
 
   const char* name();

src/comm_types/comm_serial.h

@@ -52,7 +52,6 @@
       }
 #endif
 
-
 #if !defined(MODULES_OPTION_CHECK) && !defined(COMM_MODULES_INSTANTIATION)
 #ifndef COMM_SERIAL_H
 #define COMM_SERIAL_H

src/examinimd.cpp

@@ -221,7 +221,7 @@ void ExaMiniMD::run(int nsteps) {
         neighbor->create_neigh_list(system,binning,force->half_neigh,false);
       neigh_time += neigh_timer.seconds();
     } else {
-      // Exchange Halo
+      // Exchange Halo data
       comm_timer.reset();
       comm->update_halo();
       comm_time += comm_timer.seconds();

src/force_types/CMakeLists.txt

@@ -1,11 +1,16 @@
 FILE(GLOB SRCS *.cpp)
 
-
-#Skip lj_ideal, snap and cell
+#Skip snap and cell
 #TODO: SNAP is outdates and should likely be removed all together as it is
 list(FILTER SRCS EXCLUDE REGEX ".*lj_cell\\.cpp$")
-list(FILTER SRCS EXCLUDE REGEX ".*lj_idial_neigh\\.cpp$")
 list(FILTER SRCS EXCLUDE REGEX ".*snap_neigh\\.cpp$")
 
-target_sources(ExaMiniMD PRIVATE ${SRCS})
+# Skip force-type module if Kokkos Remote Spaces is not enabled
+if (ENABLE_KOKKOS_REMOTE_SPACES)
+    message(STATUS "Building with support for force_lj_neigh_distrib")
+else()
+    #Otherwise exclude
+    list(FILTER SRCS EXCLUDE REGEX ".*distrib\\.cpp$")
+endif()
 
+target_sources(ExaMiniMD PRIVATE ${SRCS})

src/force_types/force_lj_neigh.cpp

@@ -37,7 +37,6 @@
 //************************************************************************
 
 #include<force_lj_neigh_impl.h>
-
 #define FORCETYPE_DECLARE_TEMPLATE_MACRO(NeighType) ForceLJNeigh<NeighType>
 #define FORCE_MODULES_TEMPLATE
 #include<modules_neighbor.h>

src/force_types/force_lj_neigh.h

@@ -67,27 +67,16 @@ class ForceLJNeigh: public Force {
 private:
   int N_local,ntypes;
   t_x_const_rnd x;
-
-  #ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
-  t_x_shmem x_shmem;
-  t_x_shmem_local x_shmem_local;
-  #endif
   t_f f;
   t_f_atomic f_a;
   t_id id;
-  t_index global_index;
   t_type_const_rnd type;
-
-  T_X_FLOAT domain_x, domain_y, domain_z;
-  int proc_rank;
-
   Binning::t_bincount bin_count;
   Binning::t_binoffsets bin_offsets;
   T_INT nbinx,nbiny,nbinz,nhalo;
   int step;
   bool use_stackparams;
 
-
   typedef Kokkos::View<T_F_FLOAT**> t_fparams;
   typedef Kokkos::View<const T_F_FLOAT**,
       Kokkos::MemoryTraits<Kokkos::RandomAccess>> t_fparams_rnd;
@@ -126,8 +115,6 @@ class ForceLJNeigh: public Force {
   typedef Kokkos::RangePolicy<TagFullNeighPE<true>,Kokkos::IndexType<T_INT> > t_policy_full_neigh_pe_stackparams;
   typedef Kokkos::RangePolicy<TagHalfNeighPE<true>,Kokkos::IndexType<T_INT> > t_policy_half_neigh_pe_stackparams;
 
-  struct TagCopyLocalXShmem {};
-
   ForceLJNeigh (char** args, System* system, bool half_neigh_);
 
   void init_coeff(int nargs, char** args);
@@ -151,9 +138,6 @@ class ForceLJNeigh: public Force {
   KOKKOS_INLINE_FUNCTION
   void operator() (TagHalfNeighPE<STACKPARAMS>, const T_INT& i, T_V_FLOAT& PE) const;
 
-  KOKKOS_INLINE_FUNCTION
-  void operator() (TagCopyLocalXShmem, const T_INT& i) const;
-
   const char* name();
 };
 

src/force_types/force_lj_neigh_distrib.cpp

@@ -0,0 +1,43 @@
+//************************************************************************
+//  ExaMiniMD v. 1.0
+//  Copyright (2018) National Technology & Engineering Solutions of Sandia,
+//  LLC (NTESS).
+//
+//  Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
+//  retains certain rights in this software.
+//
+//  ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
+//  use in source and binary forms, with or without modification, are
+//  permitted provided that the following conditions are met:
+//
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//
+//    2. Redistributions in binary form must reproduce the above copyright notice,
+//       this list of conditions and the following disclaimer in the documentation
+//       and/or other materials provided with the distribution.
+//
+//    3. Neither the name of the Corporation nor the names of the contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
+//  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+//  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+//  IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+//  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+//  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+//  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+//  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+//  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  POSSIBILITY OF SUCH DAMAGE.
+//
+//  Questions? Contact Christian R. Trott ([email protected])
+//************************************************************************
+
+#include<force_lj_neigh_distrib_impl.h>
+#define FORCETYPE_DECLARE_TEMPLATE_MACRO(NeighType) ForceLJNeigh<NeighType>
+#define FORCE_MODULES_TEMPLATE
+#include<modules_neighbor.h>
+#undef FORCE_MODULES_TEMPLATE