diff --git a/examples/collection/do_flops.cc b/examples/collection/do_flops.cc index 8f8098200f..12b001da89 100644 --- a/examples/collection/do_flops.cc +++ b/examples/collection/do_flops.cc @@ -57,7 +57,7 @@ #include static constexpr std::size_t const default_nrow_object = 8; -static constexpr std::size_t const default_num_objs = 4; +static constexpr std::size_t const default_num_objs = 1; static constexpr double const default_tol = 1.0e-02; static constexpr std::size_t const default_flops_per_iter = 100000; @@ -84,6 +84,16 @@ do_flops( int n ) dummy( ( void * ) &c ); } +double pi(uint64_t n) { + double sum = 0.0; + int sign = 1; + for (int i = 0; i < n; ++i) { + sum += sign/(2.0*i+1.0); + sign *= -1; + } + return 4.0*sum; +} + struct NodeObj { bool is_finished_ = false; void workFinishedHandler() { is_finished_ = true; } @@ -135,21 +145,31 @@ struct GenericWork : vt::Collection { void doIteration() { iter_ += 1; + fmt::print("-- Starting Iteration --\n"); - // vt::theContext()->getTask()->startPAPIMetrics(); + vt::theContext()->getTask()->startPAPIMetrics(); - do_flops(flopsPerIter_); + // ---------------------------------------------------------- + // test non packed double precision floating point operations + // should result in ~4*n of these operations - // vt::theContext()->getTask()->stopPAPIMetrics(); - // auto res = vt::theContext()->getTask()->getPAPIMetrics(); - // for (auto [name, value] : res) { - // fmt::print(" {}: {}\n", name, value); - // } + double p; + p = pi(10000000); + fmt::print("pi: {}\n", p); + // ---------------------------------------------------------- auto proxy = this->getCollectionProxy(); proxy.reduce<&GenericWork::checkCompleteCB, vt::collective::MaxOp>( proxy[0], 0.0 ); + + vt::theContext()->getTask()->stopPAPIMetrics(); + std::unordered_map res = vt::theContext()->getTask()->getPAPIMetrics(); + for (auto [name, value] : res) { + fmt::print(" {}: {}\n", name, value); + } + + fmt::print("-- Stopping Iteration --\n"); } struct VecMsg : vt::CollectionMessage { @@ -187,8 +207,6 @@ struct GenericWork : vt::Collection { return; } - vt::theContext()->getTask()->startPAPIMetrics(); - vt::IdxBase const myIdx = getIndex().x(); auto proxy = this->getCollectionProxy(); @@ -204,12 +222,6 @@ struct GenericWork : vt::Collection { myIdx ); } - - vt::theContext()->getTask()->stopPAPIMetrics(); - auto res = vt::theContext()->getTask()->getPAPIMetrics(); - for (auto [name, value] : res) { - fmt::print(" {}: {}\n", name, value); - } } void init() { diff --git a/src/vt/context/context.cc b/src/vt/context/context.cc index 488c983c9b..347648c754 100644 --- a/src/vt/context/context.cc +++ b/src/vt/context/context.cc @@ -98,15 +98,10 @@ Context::Context([[maybe_unused]] bool const is_interop, MPI_Comm comm) { if (retval != PAPI_VER_CURRENT) handle_papi_error(retval); - /* Check for possible failures */ - if (retval < 0) + /* Enable and initialize multiplex support */ + retval = PAPI_multiplex_init(); + if (retval != PAPI_OK) handle_papi_error(retval); - - /* Print PAPI Version */ - fprintf(stdout, "PAPI Version Number\n"); - fprintf(stdout, "MAJOR: %d\n", PAPI_VERSION_MAJOR(retval)); - fprintf(stdout, "MINOR: %d\n", PAPI_VERSION_MINOR(retval)); - fprintf(stdout, "REVISION: %d\n", PAPI_VERSION_REVISION(retval)); } Context::~Context() { diff --git a/src/vt/context/runnable_context/lb_data.cc b/src/vt/context/runnable_context/lb_data.cc index 7d8d510617..40254fedcb 100644 --- a/src/vt/context/runnable_context/lb_data.cc +++ b/src/vt/context/runnable_context/lb_data.cc @@ -53,21 +53,6 @@ void LBData::start(TimeType time) { } } -void LBData::startPAPIMetrics() { - /* -- PAPI START -- */ - /* Start counting events in the Event Set */ - papi_retval_ = PAPI_start(EventSet_); - if (papi_retval_ != PAPI_OK) - handle_papi_error(papi_retval_, "LBData start: Starting counting events in the Event Set: "); - - start_real_cycles_ = PAPI_get_real_cyc(); - start_real_usec_ = PAPI_get_real_usec(); - start_virt_cycles_ = PAPI_get_virt_cyc(); - start_virt_usec_ = PAPI_get_virt_usec(); - - /* ---------------- */ -} - void LBData::finish(TimeType time) { // record end time if (should_instrument_) { @@ -75,21 +60,6 @@ void LBData::finish(TimeType time) { } } -void LBData::stopPAPIMetrics() { - /* -- PAPI STOP -- */ - /* Stop the counting of events in the Event Set */ - papi_retval_ = PAPI_stop(EventSet_, papi_values_); - if (papi_retval_ != PAPI_OK) - handle_papi_error(papi_retval_, "LBData finish: Stoping the counting of events in the Event Set: "); - - end_real_cycles_ = PAPI_get_real_cyc(); - end_real_usec_ = PAPI_get_real_usec(); - end_virt_cycles_ = PAPI_get_virt_cyc(); - end_virt_usec_ = PAPI_get_virt_usec(); - - /* -------------- */ -} - void LBData::send(elm::ElementIDStruct dest, MsgSizeType bytes) { if (should_instrument_) { lb_data_->sendToEntity(dest, cur_elm_id_, bytes); @@ -108,15 +78,15 @@ typename LBData::ElementIDStruct const& LBData::getCurrentElementID() const { return cur_elm_id_; } -std::unordered_map LBData::getPAPIMetrics() { - std::unordered_map papi_metrics = {}; - for (size_t i = 0; i < native_events_.size(); i++) { - papi_metrics[native_events_[i]] = papi_values_[i]; +std::unordered_map LBData::getPAPIMetrics() { + std::unordered_map papi_metrics = {}; + for (size_t i = 0; i < papiData_->native_events.size(); i++) { + papi_metrics[papiData_->native_events[i]] = papiData_->values[i]; } - papi_metrics[std::string("real_time")] = end_real_usec_ - start_real_usec_; - papi_metrics[std::string("real_cycles")] = end_real_cycles_ - start_real_cycles_; - papi_metrics[std::string("virt_time")] = end_virt_usec_ - start_virt_usec_; - papi_metrics[std::string("virt_cycles")] = end_virt_cycles_ - start_virt_cycles_; + papi_metrics[std::string("real_time")] = papiData_->end_real_usec - papiData_->start_real_usec; + papi_metrics[std::string("real_cycles")] = papiData_->end_real_cycles - papiData_->start_real_cycles; + papi_metrics[std::string("virt_time")] = papiData_->end_virt_usec - papiData_->start_virt_usec; + papi_metrics[std::string("virt_cycles")] = papiData_->end_virt_cycles - papiData_->start_virt_cycles; // for (auto [name, value] : papi_metrics) { // fmt::print("{}: {}\n", name, value); // } diff --git a/src/vt/context/runnable_context/lb_data.h b/src/vt/context/runnable_context/lb_data.h index 0746e2fca5..b7588619a3 100644 --- a/src/vt/context/runnable_context/lb_data.h +++ b/src/vt/context/runnable_context/lb_data.h @@ -46,6 +46,7 @@ #include "vt/vrt/collection/balance/lb_common.h" #include "vt/elm/elm_lb_data.fwd.h" +#include "vt/context/runnable_context/papi_data.h" #include @@ -60,12 +61,6 @@ struct LBData { using ElementIDStruct = elm::ElementIDStruct; using ElementLBData = elm::ElementLBData; - void handle_papi_error (int retval, std::string info) const - { - printf("%s: PAPI error %d: %s\n", info.c_str(), retval, PAPI_strerror(retval)); - exit(1); - } - LBData() = default; /** @@ -86,29 +81,9 @@ struct LBData { LBData(ElementLBData* in_lb_data, ElementIDStruct const& in_elm_id) : lb_data_(in_lb_data), cur_elm_id_(in_elm_id), - should_instrument_(true) - { - /* Create the PAPI Event Set */ - papi_retval_ = PAPI_create_eventset(&EventSet_); - if (papi_retval_ != PAPI_OK) { - printf("LBData Constructor 2: Creating the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_)); - exit(1); - } - - for (const auto& event_name : native_events_) { - int native = 0x0; - papi_retval_ = PAPI_event_name_to_code(event_name.c_str(), &native); - if (papi_retval_ != PAPI_OK) { - printf("LBData Constructor 1: Couldn't event_name_to_code for %s: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_)); - exit(1); - } - papi_retval_ = PAPI_add_event(EventSet_, native); - if (papi_retval_ != PAPI_OK) { - printf("LBData Constructor 1: Couldn't add %s to the PAPI Event Set: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_)); - exit(1); - } - } - } + should_instrument_(true), + papiData_(std::make_unique()) + { } /** * \brief Return whether time is required @@ -147,7 +122,7 @@ struct LBData { /** * \brief Start PAPI metrics map for the running context */ - void startPAPIMetrics(); + void startPAPIMetrics() { papiData_->start(); } /** * \brief Stop PAPI metrics map for the running context @@ -155,25 +130,21 @@ struct LBData { * \note has to be called after startPAPIMetrics * */ - void stopPAPIMetrics(); + void stopPAPIMetrics() { papiData_->stop(); } /** * \brief Get the current PAPI metrics map for the running context * * \return the PAPI metrics map */ - std::unordered_map getPAPIMetrics(); + std::unordered_map getPAPIMetrics(); private: ElementLBData* lb_data_ = nullptr; /**< Element LB data */ ElementIDStruct cur_elm_id_ = {}; /**< Current element ID */ bool should_instrument_ = false; /**< Whether we are instrumenting */ int EventSet_ = PAPI_NULL; - int papi_retval_; - long long start_real_cycles_, end_real_cycles_, start_real_usec_, end_real_usec_; - long long start_virt_cycles_, end_virt_cycles_, start_virt_usec_, end_virt_usec_; - std::vector native_events_ = {"instructions", "cache-misses", "fp_arith_inst_retired.scalar_double"}; - long_long papi_values_[3]; + std::unique_ptr papiData_; }; }} /* end namespace vt::ctx */ diff --git a/src/vt/context/runnable_context/lb_data.impl.h b/src/vt/context/runnable_context/lb_data.impl.h index 5ceb03e6e7..d7eb77f2e5 100644 --- a/src/vt/context/runnable_context/lb_data.impl.h +++ b/src/vt/context/runnable_context/lb_data.impl.h @@ -58,31 +58,11 @@ template LBData::LBData(ElmT* in_elm, MsgT* msg) : lb_data_(&in_elm->getLBData()), cur_elm_id_(in_elm->getElmID()), - should_instrument_(msg->lbLiteInstrument()) + should_instrument_(msg->lbLiteInstrument()), + papiData_(std::make_unique()) { // record the communication LB data right away! theCollection()->recordLBData(in_elm, msg); - - /* Create the PAPI Event Set */ - papi_retval_ = PAPI_create_eventset(&EventSet_); - if (papi_retval_ != PAPI_OK) { - printf("LBData Constructor 1: Creating the PAPI Event Set: PAPI error %d: %s\n", papi_retval_, PAPI_strerror(papi_retval_)); - exit(1); - } - - for (const auto& event_name : native_events_) { - int native = 0x0; - papi_retval_ = PAPI_event_name_to_code(event_name.c_str(), &native); - if (papi_retval_ != PAPI_OK) { - printf("LBData Constructor 2: Couldn't event_name_to_code for %s: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_)); - exit(1); - } - papi_retval_ = PAPI_add_event(EventSet_, native); - if (papi_retval_ != PAPI_OK) { - printf("LBData Constructor 2: Couldn't add %s to the PAPI Event Set: PAPI error %d: %s\n",event_name.c_str(), papi_retval_, PAPI_strerror(papi_retval_)); - exit(1); - } - } } }} /* end namespace vt::ctx */ diff --git a/src/vt/context/runnable_context/papi_data.h b/src/vt/context/runnable_context/papi_data.h new file mode 100644 index 0000000000..f928bdd88c --- /dev/null +++ b/src/vt/context/runnable_context/papi_data.h @@ -0,0 +1,142 @@ +/* +//@HEADER +// ***************************************************************************** +// +// papi_data.h +// DARMA/vt => Virtual Transport +// +// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_VT_CONTEXT_RUNNABLE_CONTEXT_LB_DATA_PAPI_DATA_H +#define INCLUDED_VT_CONTEXT_RUNNABLE_CONTEXT_LB_DATA_PAPI_DATA_H + +#include + +namespace vt { namespace ctx { + +/** + * \struct PAPIData + * + * \brief Structure for storing Performance API (PAPI) related data structures + */ +struct PAPIData { + int EventSet = PAPI_NULL; + int retval = 0; + uint64_t start_real_cycles = 0, end_real_cycles = 0, start_real_usec = 0, end_real_usec = 0; + uint64_t start_virt_cycles = 0, end_virt_cycles = 0, start_virt_usec = 0, end_virt_usec = 0; + std::vector native_events = { + "fp_arith_inst_retired.scalar_double", + "cache-misses", + "page-faults" + }; + uint64_t values[4]; + + PAPIData() + { + std::fill(std::begin(values), std::end(values), 0); + /* Create an EventSet */ + retval = PAPI_create_eventset(&(EventSet)); + if (retval != PAPI_OK) + handle_error("PAPIData constructor: Couldn't create an event set: "); + + /* Add Total Instructions to our EventSet, + must be done such that we can call PAPI_set_multiplex */ + int native = 0x0; + retval = PAPI_event_name_to_code("instructions", &native); + retval = PAPI_add_event(EventSet, native); + if (retval != PAPI_OK) + handle_error("PAPIData constructor: Couldn't add instructions to event set: "); + + /* Convert the EventSet to a multiplexed EventSet */ + retval = PAPI_set_multiplex(EventSet); + if (retval != PAPI_OK) + handle_error("PAPIData constructor: Couldn't convert event set to multiplexed: "); + + for (const auto& event_name : native_events) { + native = 0x0; + retval = PAPI_event_name_to_code(event_name.c_str(), &native); + if (retval != PAPI_OK) { + printf("PAPIData constructor: Couldn't event_name_to_code for %s: PAPI error %d: %s\n",event_name.c_str(), retval, PAPI_strerror(retval)); + exit(1); + } + retval = PAPI_add_event(EventSet, native); + if (retval != PAPI_OK) { + printf("PAPIData constructor: Couldn't add %s to the PAPI Event Set: PAPI error %d: %s\n",event_name.c_str(), retval, PAPI_strerror(retval)); + exit(1); + } + } + // we added instructions on its own to the event set for multiplex initialization purposes; + // here we add it to the list of measured events for later usage + native_events.insert(native_events.begin(), "instructions"); + } + + void handle_error (std::string info) const + { + printf("%s: PAPI error %d: %s\n", info.c_str(), retval, PAPI_strerror(retval)); + exit(1); + } + + void start() + { + retval = PAPI_start(EventSet); + if (retval != PAPI_OK) + handle_error("PAPIData start: Starting counting events in the Event Set: "); + + start_real_cycles = PAPI_get_real_cyc(); + start_real_usec = PAPI_get_real_usec(); + start_virt_cycles = PAPI_get_virt_cyc(); + start_virt_usec = PAPI_get_virt_usec(); + } + + void stop() + { + retval = PAPI_stop(EventSet, reinterpret_cast(values)); + if (retval != PAPI_OK) + handle_error("PAPIData stop: Stopping the counting of events in the Event Set: "); + + end_real_cycles = PAPI_get_real_cyc(); + end_real_usec = PAPI_get_real_usec(); + end_virt_cycles = PAPI_get_virt_cyc(); + end_virt_usec = PAPI_get_virt_usec(); + } + + +}; + +}} /* end namespace vt::ctx */ + +#endif /*INCLUDED_VT_CONTEXT_RUNNABLE_CONTEXT_LB_DATA_PAPI_DATA_H*/ diff --git a/src/vt/runnable/runnable.cc b/src/vt/runnable/runnable.cc index ff06007ec4..72d94c7842 100644 --- a/src/vt/runnable/runnable.cc +++ b/src/vt/runnable/runnable.cc @@ -199,8 +199,8 @@ void RunnableNew::run() { #endif } -std::unordered_map RunnableNew::getPAPIMetrics() { - std::unordered_map result = {}; +std::unordered_map RunnableNew::getPAPIMetrics() { + std::unordered_map result = {}; if (contexts_.has_lb) { result = contexts_.lb.getPAPIMetrics(); diff --git a/src/vt/runnable/runnable.h b/src/vt/runnable/runnable.h index 48e300c815..69aa6e7419 100644 --- a/src/vt/runnable/runnable.h +++ b/src/vt/runnable/runnable.h @@ -339,7 +339,7 @@ struct RunnableNew { * * \return the dictionnary */ - std::unordered_map getPAPIMetrics(); + std::unordered_map getPAPIMetrics(); #if vt_check_enabled(fcontext) /**