Allowed Caliper to control Kokkos fencing behavior

src/services/kokkos/CMakeLists.txt

@@ -1,3 +1,4 @@
+include_directories(include)
 set(CALIPER_KOKKOS_SOURCES
   KokkosProfilingSymbols.cpp
   KokkosLookup.cpp

src/services/kokkos/KokkosProfilingSymbols.cpp

@@ -3,23 +3,52 @@
 #include "caliper/cali.h"
 #include "types.hpp"
 #include <cstdint>
+#include <impl/Kokkos_Profiling_Interface.hpp>
 cali::kokkos::callbacks kokkosp_callbacks;
 using cali::kokkos::SpaceHandle;
 namespace kokkos {
-    cali::ConfigManager mgr;
+cali::ConfigManager mgr;
+bool can_control_fences = false;
+bool config_needs_fences = true;
+Kokkos::Tools::Experimental::ToolProgrammingInterface kokkos_interface;
+template <typename Container, typename FenceIf>
+void fence_if_needed(const Container &in, uint32_t dev_id,
+                     const FenceIf &predicate) {
+  if (can_control_fences && !in.empty() && predicate()) {
+    kokkos_interface.fence(dev_id);
+  }
 }
-extern "C" void kokkosp_print_help(char* progName){
+} // namespace kokkos
+extern "C" void kokkosp_print_help(char *progName) {
   std::cerr << "Caliper: available configs: \n";
-  for(auto conf: kokkos::mgr.available_config_specs() ) {
-    std::cerr << kokkos::mgr.get_documentation_for_spec(conf.c_str()) << std::endl;
+  for (auto conf : kokkos::mgr.available_config_specs()) {
+    std::cerr << kokkos::mgr.get_documentation_for_spec(conf.c_str())
+              << std::endl;
+  }
+}
+extern "C" void kokkosp_request_tool_settings(
+    int num_actions, Kokkos::Tools::Experimental::ToolSettings *settings) {
+  if ((num_actions > 0) && (settings != nullptr)) {
+    settings->requires_global_fencing = false;
   }
 }
+
+extern "C" void kokkosp_provide_tool_programming_interface(
+    int num_actions,
+    Kokkos::Tools::Experimental::ToolProgrammingInterface interface) {
+  kokkos::can_control_fences = true;
+  kokkos::kokkos_interface = interface;
+}
 extern "C" void kokkosp_parse_args(int argc, char *argv_raw[]) {
   if (argc > 2) {
     std::cerr << "Error: the Kokkos Caliper connector takes only one argument"
               << std::endl;
   }
   if (argc == 2) {
+    if (std::string(argv_raw[1]).find("cuda-activity-report") !=
+        std::string::npos) {
+      kokkos::config_needs_fences = false;
+    }
     kokkos::mgr.add(argv_raw[1]);
     if (kokkos::mgr.error()) {
       std::cerr << "Kokkos Caliper connector error: " << kokkos::mgr.error_msg()
@@ -46,40 +75,62 @@ extern "C" void kokkosp_finalize_library() {
 extern "C" void kokkosp_begin_parallel_for(const char *name,
                                            const uint32_t devID,
                                            uint64_t *kID) {
+  kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_begin_parallel_for_callback,
+                          devID, [=]() { return kokkos::config_needs_fences; });
   kokkosp_callbacks.kokkosp_begin_parallel_for_callback(name, devID, kID);
+  *kID = devID;
 }
 extern "C" void kokkosp_begin_parallel_reduce(const char *name,
                                               const uint32_t devID,
                                               uint64_t *kID) {
+  kokkos::fence_if_needed(
+      kokkosp_callbacks.kokkosp_begin_parallel_reduce_callback, devID,
+      [=]() { return kokkos::config_needs_fences; });
   kokkosp_callbacks.kokkosp_begin_parallel_reduce_callback(name, devID, kID);
+  *kID = devID;
 }
 extern "C" void kokkosp_begin_parallel_scan(const char *name,
                                             const uint32_t devID,
                                             uint64_t *kID) {
+  kokkos::fence_if_needed(
+      kokkosp_callbacks.kokkosp_begin_parallel_scan_callback, devID,
+      [=]() { return kokkos::config_needs_fences; });
   kokkosp_callbacks.kokkosp_begin_parallel_scan_callback(name, devID, kID);
+  *kID = devID;
 }
 extern "C" void kokkosp_begin_fence(const char *name, const uint32_t devID,
                                     uint64_t *kID) {
   kokkosp_callbacks.kokkosp_begin_fence_callback(name, devID, kID);
 }
 
 extern "C" void kokkosp_end_parallel_for(const uint64_t kID) {
+  kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_end_parallel_for_callback,
+                          kID, [=]() { return kokkos::config_needs_fences; });
   kokkosp_callbacks.kokkosp_end_parallel_for_callback(kID);
 }
 extern "C" void kokkosp_end_parallel_reduce(const uint64_t kID) {
+  kokkos::fence_if_needed(
+      kokkosp_callbacks.kokkosp_end_parallel_reduce_callback, kID,
+      [=]() { return kokkos::config_needs_fences; });
   kokkosp_callbacks.kokkosp_end_parallel_reduce_callback(kID);
 }
 extern "C" void kokkosp_end_parallel_scan(const uint64_t kID) {
+  kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_end_parallel_scan_callback,
+                          kID, [=]() { return kokkos::config_needs_fences; });
   kokkosp_callbacks.kokkosp_end_parallel_scan_callback(kID);
 }
 extern "C" void kokkosp_end_fence(const uint64_t kID) {
   kokkosp_callbacks.kokkosp_end_fence_callback(kID);
 }
 
 extern "C" void kokkosp_push_profile_region(char *regionName) {
+  kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_push_region_callback, 0,
+                          [=]() { return true; });
   kokkosp_callbacks.kokkosp_push_region_callback(regionName);
 }
 extern "C" void kokkosp_pop_profile_region() {
+  kokkos::fence_if_needed(kokkosp_callbacks.kokkosp_push_region_callback, 0,
+                          [=]() { return true; });
   kokkosp_callbacks.kokkosp_pop_region_callback();
 }
 extern "C" void kokkosp_allocate_data(const SpaceHandle space,

src/services/kokkos/KokkosProfilingSymbols.hpp

@@ -1,9 +1,9 @@
 #ifndef CALIPER_SERVICES_KOKKOS_PROFILING_SYMBOLS_HPP
 #define CALIPER_SERVICES_KOKKOS_PROFILING_SYMBOLS_HPP
 
-
+//comment
 #include "types.hpp"
-
+#include <impl/Kokkos_Profiling_Interface.hpp>
 
 #endif