diff --git a/src/vt/configs/arguments/app_config.h b/src/vt/configs/arguments/app_config.h index 6548dc2fd1..9bd6c75d1a 100644 --- a/src/vt/configs/arguments/app_config.h +++ b/src/vt/configs/arguments/app_config.h @@ -148,6 +148,7 @@ struct AppConfig { bool vt_lb_data = false; bool vt_lb_data_compress = true; bool vt_lb_data_in = false; + uint32_t vt_lb_data_retention = 0; std::string vt_lb_data_dir = "vt_lb_data"; std::string vt_lb_data_file = "data.%p.json"; std::string vt_lb_data_dir_in = "vt_lb_data_in"; @@ -325,6 +326,7 @@ struct AppConfig { | vt_lb_interval | vt_lb_data | vt_lb_data_compress + | vt_lb_data_retention | vt_lb_data_dir | vt_lb_data_file | vt_lb_data_in diff --git a/src/vt/configs/arguments/args.cc b/src/vt/configs/arguments/args.cc index 2cde3f1710..41879594cd 100644 --- a/src/vt/configs/arguments/args.cc +++ b/src/vt/configs/arguments/args.cc @@ -911,6 +911,7 @@ void addLbArgs(CLI::App& app, AppConfig& appConfig) { auto lb_data = "Enable load balancing data"; auto lb_data_in = "Enable load balancing data input"; auto lb_data_comp = "Compress load balancing data output with brotli"; + auto lb_data_hist = "Minimal number of historical LB data phases to retain"; auto lb_data_dir = "Load balancing data output directory"; auto lb_data_file = "Load balancing data output file name"; auto lb_data_dir_in = "Load balancing data input directory"; @@ -934,6 +935,7 @@ void addLbArgs(CLI::App& app, AppConfig& appConfig) { auto ww = app.add_flag("--vt_lb_data", appConfig.vt_lb_data, lb_data); auto za = app.add_flag("--vt_lb_data_in", appConfig.vt_lb_data_in, lb_data_in); auto xz = app.add_flag("--vt_lb_data_compress", appConfig.vt_lb_data_compress, lb_data_comp); + auto dr = app.add_option("--vt_lb_data_retention", appConfig.vt_lb_data_retention, lb_data_hist); auto wx = app.add_option("--vt_lb_data_dir", appConfig.vt_lb_data_dir, lb_data_dir)->capture_default_str(); auto wy = app.add_option("--vt_lb_data_file", appConfig.vt_lb_data_file, lb_data_file)->capture_default_str(); auto xx = app.add_option("--vt_lb_data_dir_in", appConfig.vt_lb_data_dir_in, lb_data_dir_in)->capture_default_str(); @@ -967,6 +969,7 @@ void addLbArgs(CLI::App& app, AppConfig& appConfig) { xx->group(debugLB); xy->group(debugLB); xz->group(debugLB); + dr->group(debugLB); yx->group(debugLB); yy->group(debugLB); yz->group(debugLB); diff --git a/src/vt/elm/elm_lb_data.cc b/src/vt/elm/elm_lb_data.cc index 4489e6bea7..37ecfb9da4 100644 --- a/src/vt/elm/elm_lb_data.cc +++ b/src/vt/elm/elm_lb_data.cc @@ -195,7 +195,15 @@ void ElementLBData::updatePhase(PhaseType const& inc) { } void ElementLBData::resetPhase() { + // This method will become obsolete once VT gains full restart capability, + // allowing it to load all necessary data (like PhaseManager state, NodeLBData, etc.) from a checkpoint. + cur_phase_ = fst_lb_phase; + // Resets the current phase in the containers. + phase_timings_.restartFrom(fst_lb_phase); + subphase_timings_.restartFrom(fst_lb_phase); + phase_comm_.restartFrom(fst_lb_phase); + subphase_comm_.restartFrom(fst_lb_phase); } PhaseType ElementLBData::getPhase() const { @@ -203,8 +211,7 @@ PhaseType ElementLBData::getPhase() const { } LoadType ElementLBData::getLoad(PhaseType const& phase) const { - auto iter = phase_timings_.find(phase); - if (iter != phase_timings_.end()) { + if (phase_timings_.contains(phase)) { auto const total_load = phase_timings_.at(phase); vt_debug_print( @@ -276,13 +283,11 @@ SubphaseType ElementLBData::getSubPhase() const { return cur_subphase_; } -void ElementLBData::releaseLBDataFromUnneededPhases(PhaseType phase, unsigned int look_back) { - if (phase >= look_back) { - phase_timings_.erase(phase - look_back); - subphase_timings_.erase(phase - look_back); - phase_comm_.erase(phase - look_back); - subphase_comm_.erase(phase - look_back); - } +void ElementLBData::setHistoryCapacity(unsigned int hist_lb_data_count) { + phase_timings_.resize(hist_lb_data_count); + subphase_timings_.resize(hist_lb_data_count); + phase_comm_.resize(hist_lb_data_count); + subphase_comm_.resize(hist_lb_data_count); } std::size_t ElementLBData::getLoadPhaseCount() const { diff --git a/src/vt/elm/elm_lb_data.h b/src/vt/elm/elm_lb_data.h index 576753250a..40908c9feb 100644 --- a/src/vt/elm/elm_lb_data.h +++ b/src/vt/elm/elm_lb_data.h @@ -47,6 +47,7 @@ #include "vt/elm/elm_id.h" #include "vt/elm/elm_comm.h" #include "vt/timing/timing.h" +#include "vt/utils/container/circular_phases_buffer.h" namespace vt { namespace vrt { namespace collection { namespace balance { @@ -122,24 +123,23 @@ struct ElementLBData { static const constexpr SubphaseType no_subphase = std::numeric_limits::max(); -protected: /** - * \internal \brief Release LB data from phases prior to lookback + * \brief Resize internal buffers + * + * \param[in] hist_lb_data_count the requested buffers capacity */ - void releaseLBDataFromUnneededPhases(PhaseType phase, unsigned int look_back); - - friend struct vrt::collection::balance::NodeLBData; + void setHistoryCapacity(unsigned int hist_lb_data_count); protected: bool cur_time_started_ = false; TimeType cur_time_ = TimeType{0.0}; PhaseType cur_phase_ = fst_lb_phase; - std::unordered_map phase_timings_ = {}; - std::unordered_map phase_comm_ = {}; + util::container::CircularPhasesBuffer phase_timings_ = {}; + util::container::CircularPhasesBuffer phase_comm_ = {}; SubphaseType cur_subphase_ = 0; - std::unordered_map> subphase_timings_ = {}; - std::unordered_map> subphase_comm_ = {}; + util::container::CircularPhasesBuffer> subphase_timings_ = {}; + util::container::CircularPhasesBuffer> subphase_comm_ = {}; }; }} /* end namespace vt::elm */ diff --git a/src/vt/utils/container/circular_phases_buffer.h b/src/vt/utils/container/circular_phases_buffer.h new file mode 100644 index 0000000000..52aa64b9f3 --- /dev/null +++ b/src/vt/utils/container/circular_phases_buffer.h @@ -0,0 +1,432 @@ +/* +//@HEADER +// ***************************************************************************** +// +// circular_phases_buffer.h +// DARMA/vt => Virtual Transport +// +// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_VT_UTILS_CONTAINER_CIRCULAR_PHASES_BUFFER_H +#define INCLUDED_VT_UTILS_CONTAINER_CIRCULAR_PHASES_BUFFER_H + +#include "vt/config.h" +#include +#include + +namespace vt { namespace util { namespace container { + +/** + * \struct CircularPhasesBuffer + * + * \brief A circular buffer for storing and accessing data with phase-based indexing. + * + * The buffer supports a valid range of phases that spans from `frontPhase() - capacity()` + * to `frontPhase()`, allowing both random access and insertion of new phases within this window. + * Phases outside this range, i.e., those before the window, are not guaranteed to remain accessible. + * Inserting a phase greater than the current `frontPhase()` will automatically advance the + * `frontPhase()`, shifting the valid window forward to accommodate the new phase. + */ +template +class CircularPhasesBuffer { + using StoredPair = std::pair; + + constexpr static auto no_phase = std::numeric_limits::max(); + constexpr static auto no_index = std::numeric_limits::max(); + + /** + * \brief Create an empty phase-data pair for initializing the buffer. + * + * \param[in] phase The phase to assign to the pair (defaults to \c no_phase if not provided) + * \return A new pair containing the phase and an empty \c StoredType object. + */ + static StoredPair makeEmptyPair(const PhaseType& phase = no_phase) { + return std::make_pair(phase, StoredType{}); + } + +public: + + /** + * \brief Constructor to initialize the circular buffer with a specified size. + * + * \param[in] size_in The size of the buffer (must be greater than zero). + */ + CircularPhasesBuffer(std::size_t size_in = 1) + : buffer_(size_in, makeEmptyPair()) + { + vtAssert(size_in > 0, "Size of CircularPhasesBuffer needs to be greather than zero."); + } + + /** + * \brief Check if a given phase exists in the buffer. + * + * \param[in] phase The phase to check for existence. + * \return \c true if the phase is present, otherwise \c false. + */ + bool contains(const PhaseType& phase) const { + return buffer_[phaseToIndex(phase)].first == phase; + } + + /** + * \brief Access data for a given phase. If the phase is not present, it inserts a new element. + * This method can abort the process if the phase is out of range (check \c canBeStored()). + * + * \param[in] phase The phase to retrieve data for. + * \return A reference to the stored data for the phase. + */ + StoredType& operator[](const PhaseType phase) { + auto& pair = buffer_[phaseToIndex(phase)]; + // Inserts empty data for new phase + if (pair.first != phase) { + store(phase, StoredType{}); + } + return pair.second; + } + + /** + * \brief Store data for a given phase in the buffer. + * Can abort the process if the phase is out of the valid range (check \c canBeStored()). + * + * \param[in] phase The phase for which data will be stored. + * \param[in] data The data to store. + */ + void store(const PhaseType& phase, StoredType data) { + vtAssert(canBeStored(phase), "Phase is out of valid range"); + + buffer_[phaseToIndex(phase)] = std::make_pair(phase, data); + updateHead(phase); + } + + /** + * \brief Retrieve data for a specific phase. + * + * \param[in] phase The phase to retrieve. + * \return A reference to the stored data. + * \throw Asserts if the phase is not present in the buffer. + */ + StoredType& at(const PhaseType phase) { + vtAssert(contains(phase), "Phase is not stored in the buffer."); + return buffer_[phaseToIndex(phase)].second; + } + + /** + * \brief Retrieve data for a specific phase. + * + * \param[in] phase The phase to retrieve. + * \return A const reference to the stored data. + * \throw Asserts if the phase is not present in the buffer. + */ + const StoredType& at(const PhaseType phase) const { + vtAssert(contains(phase), "Phase is not stored in the buffer."); + return buffer_[phaseToIndex(phase)].second; + } + + /** + * \brief Resize the buffer to a new size. + * The buffer's minimal size is 1. + * + * \param[in] new_size_in The requested new size of the buffer. + */ + void resize(const std::size_t new_size_in) { + auto new_size = std::max(std::size_t{1}, new_size_in); + if (new_size == buffer_.size()) { + return; + } + + // temporary vector to copy the elements to retain + std::vector tmp(new_size, makeEmptyPair()); + // number of elements to copy + auto num = std::min(new_size, buffer_.size() - numFree()); + + // copy data which should be retained + auto to_copy = head_phase_; + while(num > 0 && to_copy != no_phase) { + if (contains(to_copy)) { + tmp[to_copy % tmp.size()] = buffer_[phaseToIndex(to_copy)]; + num--; + } + to_copy--; + } + + buffer_.swap(tmp); + } + + /** + * \brief Check if the buffer is empty (i.e., contains no valid phases). + * + * \return \c true if the buffer is empty, otherwise \c false. + */ + bool empty() const { + return head_phase_ == no_phase; + } + + /** + * \brief Get the number of valid elements currently stored in the buffer. + * + * \return The number of valid elements in the buffer. + */ + std::size_t size() const { + return buffer_.size() - numFree(); + } + + /** + * \brief Get the total capacity of the buffer. + * + * \return The capacity of the buffer. + */ + std::size_t capacity() const { + return buffer_.size(); + } + + /** + * \brief Clear all elements in the buffer, resetting it to an empty state. + */ + void clear() { + head_phase_ = no_phase; + buffer_.assign(buffer_.size(), makeEmptyPair()); + } + + /** + * \brief Resets the buffer to a specified phase, discarding all earlier phases + * and retaining only the most recent phase data that will be mapped to the new starting point. + * + * \param start_point The phase to start from. + */ + void restartFrom(const PhaseType& start_point) { + if (empty()) { + head_phase_ = start_point; + return; + } + + std::size_t index = phaseToIndex(start_point); + if (std::size_t head_index = phaseToIndex(head_phase_); head_index != index) { + // copy data from head_phase_ + buffer_[index].second = buffer_[head_index].second; + } + + // clear rest of the data + for (std::size_t i = 0; i < buffer_.size(); i++) { + if (i != index) { + buffer_[i] = makeEmptyPair(); + } + } + + buffer_[index].first = start_point; + head_phase_ = start_point; + } + + /** + * \brief Retrieve the most recent phase in the buffer. + * + * \return The phase at the front of the buffer. + */ + PhaseType frontPhase() const { + return head_phase_; + } + + /** + * \brief Retrieve the data associated with the most recent phase. + * + * \return A reference to the data for the latest phase. + * \throw Asserts if the buffer is empty. + */ + StoredType& frontData() { + return at(head_phase_); + } + + /** + * \brief Find data associated with a phase. + * + * \param[in] phase The phase to search for. + * \return A pointer to the data if the phase is present, otherwise \c nullptr. + */ + StoredType* find(const PhaseType& phase) { + if (contains(phase)) { + return &buffer_[phaseToIndex(phase)].second; + } + return nullptr; + } + + /** + * \brief Find data associated with a phase. + * + * \param[in] phase The phase to search for. + * \return A const pointer to the data if the phase is present, otherwise \c nullptr. + */ + const StoredType* find(const PhaseType& phase) const { + if (contains(phase)) { + return &buffer_[phaseToIndex(phase)].second; + } + return nullptr; + } + + /** + * \brief Get a const iterator to the first valid element in the buffer. + * + * \return A const iterator to the first valid element. + */ + auto begin() const { return ++ForwardIterator(&buffer_, no_index); } + + /** + * \brief Get an iterator to the first valid element in the buffer. + * + * \return An iterator to the first valid element. + */ + auto begin() { return ++ForwardIterator(&buffer_, no_index); } + + /** + * \brief Get a const iterator to the end of the buffer (one past the last element). + * + * \return A const iterator to the end of the buffer. + */ + auto end() const { return ForwardIterator(&buffer_, buffer_.size()); } + + /** + * \brief Get an iterator to the end of the buffer (one past the last element). + * + * \return An iterator to the end of the buffer. + */ + auto end() { return ForwardIterator(&buffer_, buffer_.size()); } + + template + void serialize(SerializeT& s) { + s | head_phase_; + s | buffer_; + } + +private: + /** + * \brief Convert a phase number to its corresponding index in the buffer. + * + * \param[in] phase The phase to convert. + * \return The buffer index for the phase. + */ + std::size_t phaseToIndex(const PhaseType& phase) const { + return phase % buffer_.size(); + } + + /** + * \brief Count the number of empty (without a valid phase) slots in the buffer. + * + * \return The number of empty slots. + */ + std::size_t numFree() const { + return std::count_if(buffer_.begin(), buffer_.end(), + [](const StoredPair& pair){ return pair.first == no_phase; } + ); + } + + /** + * \brief Update the head phase if a newer phase is stored. + * + * \param[in] phase The phase being stored. + */ + void updateHead(const PhaseType& phase) { + if (head_phase_ == no_phase || phase > head_phase_) { + head_phase_ = phase; + } + } + + /** + * \brief Check whether a given phase can be stored in the buffer. + * A phase is valid if it is greater than \c (head_phase_ - capacity()). + * + * \param[in] phase The phase to validate. + * \return \c true if the phase can be stored, otherwise \c false. + */ + bool canBeStored(const PhaseType& phase) const { + if (!empty() && head_phase_ > capacity()) { + return phase > head_phase_ - capacity(); + } + return true; + } + + PhaseType head_phase_ = no_phase; ///< The most recent phase in the buffer. + std::vector buffer_; ///< The underlying data buffer. + +public: + /** + * \struct ForwardIterator + * + * \brief An iterator for traversing the buffer in a forward direction, skipping empty slots. + */ + struct ForwardIterator { + using iterator_category = std::forward_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = StoredPair; + using reference = StoredPair&; + + ForwardIterator(std::vector* buffer, std::size_t index) + : index_(index), buffer_(buffer) { } + + reference operator*() { return (*buffer_)[index_]; } + + ForwardIterator& operator++() { + advance(); + return *this; + } + ForwardIterator operator++(int) { + ForwardIterator tmp = *this; + ++(*this); + return tmp; + } + + bool operator== (const ForwardIterator& it) const { return index_ == it.index_; }; + bool operator!= (const ForwardIterator& it) const { return index_ != it.index_; }; + + private: + /** + * \brief Move the iterator to the next element with a valid phase. + */ + void advance() { + index_++; + if (index_ == buffer_->size()) { + return; + } + if ((*buffer_)[index_].first == no_phase) { + advance(); + } + } + + std::size_t index_; ///< The current index in the buffer. + std::vector* buffer_; ///< Pointer to the buffer being iterated over. + }; +}; + +}}} /* end namespace vt::util::container */ + +#endif /*INCLUDED_VT_UTILS_CONTAINER_CIRCULAR_PHASES_BUFFER_H*/ diff --git a/src/vt/vrt/collection/balance/lb_common.h b/src/vt/vrt/collection/balance/lb_common.h index e9e403b2de..b2fececfe3 100644 --- a/src/vt/vrt/collection/balance/lb_common.h +++ b/src/vt/vrt/collection/balance/lb_common.h @@ -49,6 +49,7 @@ #include "vt/elm/elm_comm.h" #include "vt/timing/timing_type.h" #include "vt/messaging/message/message.h" +#include "vt/utils/container/circular_phases_buffer.h" #include @@ -63,6 +64,7 @@ namespace balance { using ElementIDStruct = elm::ElementIDStruct; using ElementIDType = elm::ElementIDType; using CommMapType = elm::CommMapType; +using CommMapBufferType = util::container::CircularPhasesBuffer; static constexpr ElementIDType const no_element_id = elm::no_element_id; @@ -117,9 +119,11 @@ using UserDataValueType = std::variant; using ElmUserDataType = std::unordered_map; using LoadMapType = std::unordered_map; +using LoadMapBufferType = util::container::CircularPhasesBuffer; using SubphaseLoadMapType = std::unordered_map>; /// User-defined LB values map using DataMapType = std::unordered_map; +using DataMapBufferType = util::container::CircularPhasesBuffer; struct Reassignment { // Include the subject node so that these structures can be formed diff --git a/src/vt/vrt/collection/balance/lb_data_holder.cc b/src/vt/vrt/collection/balance/lb_data_holder.cc index e6cba201d5..71a038627e 100644 --- a/src/vt/vrt/collection/balance/lb_data_holder.cc +++ b/src/vt/vrt/collection/balance/lb_data_holder.cc @@ -51,6 +51,8 @@ #include +#include + namespace vt { namespace vrt { namespace collection { namespace balance { void LBDataHolder::getObjectFromJsonField_( @@ -205,14 +207,14 @@ std::unique_ptr LBDataHolder::toJson(PhaseType phase) const { j["id"] = phase; std::size_t i = 0; - if (node_data_.find(phase) != node_data_.end()) { + if (node_data_.contains(phase)) { for (auto&& elm : node_data_.at(phase)) { ElementIDStruct id = elm.first; LoadType time = elm.second.whole_phase_load; j["tasks"][i]["resource"] = "cpu"; j["tasks"][i]["node"] = id.getCurrNode(); j["tasks"][i]["time"] = time; - if (user_defined_json_.find(phase) != user_defined_json_.end()) { + if (user_defined_json_.contains(phase)) { auto &user_def_this_phase = user_defined_json_.at(phase); if (user_def_this_phase.find(id) != user_def_this_phase.end()) { auto &user_def = user_def_this_phase.at(id); @@ -223,7 +225,7 @@ std::unique_ptr LBDataHolder::toJson(PhaseType phase) const { } outputEntity(j["tasks"][i]["entity"], id); - if (node_user_attributes_.find(phase) != node_user_attributes_.end()) { + if (node_user_attributes_.contains(phase)) { if (node_user_attributes_.at(phase).find(id) != node_user_attributes_.at(phase).end()) { for (auto const& [key, value] : node_user_attributes_.at(phase).at(id)) { if (std::holds_alternative(value)) { @@ -255,7 +257,7 @@ std::unique_ptr LBDataHolder::toJson(PhaseType phase) const { } i = 0; - if (node_comm_.find(phase) != node_comm_.end()) { + if (node_comm_.contains(phase)) { for (auto const& [key, volume] : node_comm_.at(phase)) { j["communications"][i]["bytes"] = volume.bytes; j["communications"][i]["messages"] = volume.messages; @@ -337,7 +339,7 @@ std::unique_ptr LBDataHolder::toTV(PhaseType phase) const { std::unordered_map objects; - if (node_data_.find(phase) != node_data_.end()) { + if (node_data_.contains(phase)) { for (auto&& elm : node_data_.at(phase)) { ElementIDStruct id = elm.first; double whole_phase_load = elm.second.whole_phase_load; @@ -345,7 +347,7 @@ std::unique_ptr LBDataHolder::toTV(PhaseType phase) const { ElmUserDataType user_defined; if ( - user_defined_lb_info_.find(phase) != user_defined_lb_info_.end() and + user_defined_lb_info_.contains(phase) and user_defined_lb_info_.at(phase).find(id) != user_defined_lb_info_.at(phase).end() ) { @@ -365,7 +367,7 @@ std::unique_ptr LBDataHolder::toTV(PhaseType phase) const { } } - if (node_comm_.find(phase) != node_comm_.end()) { + if (node_comm_.contains(phase)) { for (auto&& elm : node_comm_.at(phase)) { auto const& key = elm.first; auto const& volume = elm.second; @@ -396,7 +398,7 @@ std::unordered_map LBDataHolder::getObjInfo( PhaseType phase ) const { std::unordered_map map; - if (node_data_.find(phase) != node_data_.end()) { + if (node_data_.contains(phase)) { for (auto&& elm : node_data_.at(phase)) { ElementIDStruct id = elm.first; @@ -426,6 +428,15 @@ std::unordered_map LBDataHolder::getObjInfo( #endif +void LBDataHolder::resizeHistory(std::size_t num_to_retain) { + node_data_.resize(num_to_retain); + node_comm_.resize(num_to_retain); + node_subphase_comm_.resize(num_to_retain); + user_defined_json_.resize(num_to_retain); + user_defined_lb_info_.resize(num_to_retain); + node_user_attributes_.resize(num_to_retain); +} + LBDataHolder::LBDataHolder(nlohmann::json const& j) { this_node_ = theContext()->getNode(); @@ -439,9 +450,6 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) auto id = phase["id"]; auto tasks = phase["tasks"]; - this->node_data_[id]; - this->node_comm_[id]; - if (tasks.is_array()) { for (auto const& task : tasks) { auto node = task["node"]; @@ -632,10 +640,16 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) } void LBDataHolder::readMetadata(nlohmann::json const& j) { + std::size_t num_phases = 0; if (j.find("metadata") != j.end()) { auto metadata = j["metadata"]; if (metadata.find("phases") != metadata.end()) { auto phases = metadata["phases"]; + // read the number of phases + if (phases.find("count") != phases.end()) { + num_phases = phases["count"]; + } + // load all skipped phases auto sl = phases["skipped"]["list"]; if(sl.is_array()) { @@ -682,6 +696,34 @@ void LBDataHolder::readMetadata(nlohmann::json const& j) { } } } + + // Adjust the capacity of the data containers + if (num_phases > 0) { + resizeHistory(num_phases); + } else { + // find min and max phase + PhaseType min = std::numeric_limits::max(); + PhaseType max = 0; + + if (identical_phases_.size() > 0) { + min = std::min(min, *identical_phases_.begin()); + max = std::max(max, *identical_phases_.rbegin()); + } + + auto phases = j["phases"]; + if (phases.is_array()) { + for (auto const& phase : phases) { + auto id = phase["id"]; + if (id < min) { + min = id; + } else if (id > max) { + max = id; + } + } + } + + resizeHistory((max - min) + 1); + } } void LBDataHolder::clear() { diff --git a/src/vt/vrt/collection/balance/lb_data_holder.h b/src/vt/vrt/collection/balance/lb_data_holder.h index 618fe52a42..4451bfc94c 100644 --- a/src/vt/vrt/collection/balance/lb_data_holder.h +++ b/src/vt/vrt/collection/balance/lb_data_holder.h @@ -137,6 +137,13 @@ struct LBDataHolder { */ void clear(); + /** + * \brief Resize internal buffers to hold specified amount of phases. + * + * \param[in] num_to_retain the number of phases to retain + */ + void resizeHistory(std::size_t num_to_retain); + private: /** * \brief Output an entity to json @@ -184,21 +191,21 @@ struct LBDataHolder { /// Node attributes for the current rank ElmUserDataType rank_attributes_; /// Node timings for each local object - std::unordered_map node_data_; + LoadMapBufferType node_data_; /// Node communication graph for each local object - std::unordered_map node_comm_; + CommMapBufferType node_comm_; /// Node communication graph for each subphase - std::unordered_map> node_subphase_comm_; + util::container::CircularPhasesBuffer> node_subphase_comm_; /// User-defined data from each phase for JSON output - std::unordered_map >> user_defined_json_; std::unordered_map> user_per_phase_json_; /// User-defined data from each phase for LB - std::unordered_map user_defined_lb_info_; + DataMapBufferType user_defined_lb_info_; /// User-defined attributes from each phase - std::unordered_map node_user_attributes_; + DataMapBufferType node_user_attributes_; /// Node indices for each ID along with the proxy ID std::unordered_map>> node_idx_; /// Map from id to objgroup proxy diff --git a/src/vt/vrt/collection/balance/lb_data_restart_reader.cc b/src/vt/vrt/collection/balance/lb_data_restart_reader.cc index be21ad319b..8af49b55d4 100644 --- a/src/vt/vrt/collection/balance/lb_data_restart_reader.cc +++ b/src/vt/vrt/collection/balance/lb_data_restart_reader.cc @@ -79,18 +79,8 @@ void LBDataRestartReader::startup() { } void LBDataRestartReader::readHistory(LBDataHolder const& lbdh) { - auto find_max_data_phase = [&]() -> PhaseType { - if (lbdh.node_data_.empty()) { - return 0; - } - return std::max_element( - lbdh.node_data_.begin(), lbdh.node_data_.end(), - [](const auto& p1, const auto& p2) { return p1.first < p2.first; }) - ->first; - }; - // Find last phase number - auto largest_data = find_max_data_phase(); + auto largest_data = lbdh.node_data_.frontPhase(); auto largest_identical = lbdh.identical_phases_.size() > 0 ? *lbdh.identical_phases_.rbegin() : 0; auto largest_skipped = @@ -100,10 +90,9 @@ void LBDataRestartReader::readHistory(LBDataHolder const& lbdh) { PhaseType last_found_phase = 0; for (PhaseType phase = 0; phase < num_phases_; phase++) { - auto iter = lbdh.node_data_.find(phase); - if (iter != lbdh.node_data_.end()) { + if (lbdh.node_data_.contains(phase)) { last_found_phase = phase; - for (auto const& obj : iter->second) { + for (auto const& obj : lbdh.node_data_.at(phase)) { if (obj.first.isMigratable()) { if (history_[phase] == nullptr) { history_[phase] = std::make_shared>(); diff --git a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc index ce20121e22..3d0dcc3890 100644 --- a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc +++ b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc @@ -174,8 +174,11 @@ LBType LBManager::decideLBToRun(PhaseType phase, bool try_file) { } void LBManager::setLoadModel(std::shared_ptr model) { - model_ = model; auto nlb_data = theNodeLBData(); + min_hist_lb_data_ = std::max(model->getNumPastPhasesNeeded(), theConfig()->vt_lb_data_retention); + nlb_data->resizeLBDataHistory(min_hist_lb_data_); + + model_ = model; model_->setLoads(nlb_data->getNodeLoad(), nlb_data->getNodeComm(), nlb_data->getUserData()); @@ -254,15 +257,15 @@ LBManager::runLB(PhaseType phase, vt::Callback cb) { elm::CommMapType const* comm = &empty_comm; auto const& node_comm = theNodeLBData()->getNodeComm(); - if (auto iter = node_comm->find(phase); iter != node_comm->end()) { - comm = &iter->second; + if (node_comm->contains(phase)) { + comm = &node_comm->at(phase); } balance::DataMapType empty_data_map; balance::DataMapType const* data_map = &empty_data_map; auto const& node_data_map = theNodeLBData()->getUserData(); - if (auto iter = node_data_map->find(phase); iter != node_data_map->end()) { - data_map = &iter->second; + if (node_data_map->contains(phase)) { + data_map = &node_data_map->at(phase); } vt_debug_print(terse, lb, "LBManager: running strategy\n"); @@ -539,7 +542,7 @@ void LBManager::finishedLB(PhaseType phase) { "finishedLB\n" ); - theNodeLBData()->startIterCleanup(phase, model_->getNumPastPhasesNeeded()); + theNodeLBData()->startIterCleanup(); theNodeLBData()->outputLBDataForPhase(phase); destroyLB(); @@ -732,9 +735,8 @@ void LBManager::computeStatistics( elm::CommMapType empty_comm; elm::CommMapType const* comm_data = &empty_comm; - auto iter = theNodeLBData()->getNodeComm()->find(phase); - if (iter != theNodeLBData()->getNodeComm()->end()) { - comm_data = &iter->second; + if (theNodeLBData()->getNodeComm()->contains(phase)) { + comm_data = &theNodeLBData()->getNodeComm()->at(phase); } std::vector lstats; @@ -894,12 +896,11 @@ getSharedEdges(elm::CommMapType const& comm_data) { void makeGraphSymmetric( PhaseType phase, objgroup::proxy::Proxy proxy ) { - auto iter = theNodeLBData()->getNodeComm()->find(phase); - if (iter == theNodeLBData()->getNodeComm()->end()) { + if (!theNodeLBData()->getNodeComm()->contains(phase)) { return; } - auto shared_edges = getSharedEdges(iter->second); + auto shared_edges = getSharedEdges(theNodeLBData()->getNodeComm()->at(phase)); for (auto&& elm : shared_edges) { proxy[elm.first].send( diff --git a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.h b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.h index 446d8b9498..fd3ad4fcfa 100644 --- a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.h +++ b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.h @@ -213,7 +213,8 @@ struct LBManager : runtime::component::Component { | model_ | lb_instances_ | stats - | created_lbstats_dir_; + | created_lbstats_dir_ + | min_hist_lb_data_; } void stagePreLBStatistics(const StatisticMapType &statistics); @@ -307,6 +308,8 @@ struct LBManager : runtime::component::Component { std::unique_ptr statistics_writer_ = nullptr; /// Whether the LB statistics directory has been created bool created_lbstats_dir_ = false; + //// The amount of phases of historical LB data to hold + uint32_t min_hist_lb_data_ = 0; }; void makeGraphSymmetric( diff --git a/src/vt/vrt/collection/balance/model/comm_overhead.cc b/src/vt/vrt/collection/balance/model/comm_overhead.cc index 8b9558f2a8..d040b626d6 100644 --- a/src/vt/vrt/collection/balance/model/comm_overhead.cc +++ b/src/vt/vrt/collection/balance/model/comm_overhead.cc @@ -54,9 +54,9 @@ CommOverhead::CommOverhead( per_byte_weight_(in_per_byte_weight) { } -void CommOverhead::setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) { +void CommOverhead::setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) { proc_comm_ = proc_comm; ComposedModel::setLoads(proc_load, proc_comm, user_data); } diff --git a/src/vt/vrt/collection/balance/model/comm_overhead.h b/src/vt/vrt/collection/balance/model/comm_overhead.h index 69afe8f59f..148648ea42 100644 --- a/src/vt/vrt/collection/balance/model/comm_overhead.h +++ b/src/vt/vrt/collection/balance/model/comm_overhead.h @@ -65,14 +65,14 @@ struct CommOverhead : public ComposedModel { LoadType in_per_byte_weight ); - void setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) override; + void setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) override; LoadType getModeledLoad(ElementIDStruct object, PhaseOffset when) const override; private: - std::unordered_map const* proc_comm_; /**< Underlying comm data */ + CommMapBufferType const* proc_comm_; /**< Underlying comm data */ LoadType per_msg_weight_ = 0.001; /**< Cost per message */ LoadType per_byte_weight_ = 0.000001; /**< Cost per bytes */ }; // class CommOverhead diff --git a/src/vt/vrt/collection/balance/model/composed_model.cc b/src/vt/vrt/collection/balance/model/composed_model.cc index c827a43af3..aac29b5bdf 100644 --- a/src/vt/vrt/collection/balance/model/composed_model.cc +++ b/src/vt/vrt/collection/balance/model/composed_model.cc @@ -45,9 +45,9 @@ namespace vt { namespace vrt { namespace collection { namespace balance { -void ComposedModel::setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) { +void ComposedModel::setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) { base_->setLoads(proc_load, proc_comm, user_data); } diff --git a/src/vt/vrt/collection/balance/model/composed_model.h b/src/vt/vrt/collection/balance/model/composed_model.h index a052983703..f0e23a9a1b 100644 --- a/src/vt/vrt/collection/balance/model/composed_model.h +++ b/src/vt/vrt/collection/balance/model/composed_model.h @@ -64,9 +64,9 @@ class ComposedModel : public LoadModel // \param[in] base must not be null explicit ComposedModel(std::shared_ptr base) : base_(base) {} - void setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) override; + void setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) override; void updateLoads(PhaseType last_completed_phase) override; diff --git a/src/vt/vrt/collection/balance/model/load_model.h b/src/vt/vrt/collection/balance/model/load_model.h index 84b961e7fd..3019bbb1de 100644 --- a/src/vt/vrt/collection/balance/model/load_model.h +++ b/src/vt/vrt/collection/balance/model/load_model.h @@ -48,6 +48,7 @@ #include "vt/timing/timing_type.h" #include "vt/vrt/collection/balance/lb_common.h" #include "vt/elm/elm_comm.h" +#include "vt/utils/container/circular_phases_buffer.h" namespace vt { namespace vrt { namespace collection { namespace balance { @@ -195,9 +196,9 @@ struct LoadModel * passed a new model instance for a collection */ virtual void setLoads( - std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data + LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data ) = 0; /** diff --git a/src/vt/vrt/collection/balance/model/per_collection.cc b/src/vt/vrt/collection/balance/model/per_collection.cc index e0ce53b6cd..ce94e2cf45 100644 --- a/src/vt/vrt/collection/balance/model/per_collection.cc +++ b/src/vt/vrt/collection/balance/model/per_collection.cc @@ -55,9 +55,9 @@ void PerCollection::addModel(CollectionID proxy, std::shared_ptr mode models_[proxy] = model; } -void PerCollection::setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) { +void PerCollection::setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) { for (auto& m : models_) m.second->setLoads(proc_load, proc_comm, user_data); ComposedModel::setLoads(proc_load, proc_comm, user_data); diff --git a/src/vt/vrt/collection/balance/model/per_collection.h b/src/vt/vrt/collection/balance/model/per_collection.h index f32e69814e..a2a57d7dcc 100644 --- a/src/vt/vrt/collection/balance/model/per_collection.h +++ b/src/vt/vrt/collection/balance/model/per_collection.h @@ -73,9 +73,9 @@ struct PerCollection : public ComposedModel */ void addModel(CollectionID proxy, std::shared_ptr model); - void setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) override; + void setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) override; void updateLoads(PhaseType last_completed_phase) override; diff --git a/src/vt/vrt/collection/balance/model/raw_data.cc b/src/vt/vrt/collection/balance/model/raw_data.cc index d66eb9408d..f6dac5ca45 100644 --- a/src/vt/vrt/collection/balance/model/raw_data.cc +++ b/src/vt/vrt/collection/balance/model/raw_data.cc @@ -50,9 +50,9 @@ void RawData::updateLoads(PhaseType last_completed_phase) { last_completed_phase_ = last_completed_phase; } -void RawData::setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) +void RawData::setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) { proc_load_ = proc_load; proc_comm_ = proc_comm; @@ -60,19 +60,18 @@ void RawData::setLoads(std::unordered_map const* proc_lo } ObjectIterator RawData::begin() const { - auto iter = proc_load_->find(last_completed_phase_); - if (iter != proc_load_->end()) { - return {std::make_unique(iter->second.cbegin(), - iter->second.cend())}; + if (proc_load_->contains(last_completed_phase_)) { + auto ptr = proc_load_->find(last_completed_phase_); + return {std::make_unique(ptr->cbegin(), + ptr->cend())}; } else { return {nullptr}; } } int RawData::getNumObjects() const { - auto iter = proc_load_->find(last_completed_phase_); - if (iter != proc_load_->end()) { - return iter->second.size(); + if (proc_load_->contains(last_completed_phase_)) { + return proc_load_->at(last_completed_phase_).size(); } else { return 0; } @@ -118,7 +117,7 @@ ElmUserDataType RawData::getUserData(ElementIDStruct object, PhaseOffset offset) "RawData makes no predictions. Compose with NaivePersistence or some longer-range forecasting model as needed"); auto phase = getNumCompletedPhases() + offset.phases; - if (user_data_->find(phase) != user_data_->end()) { + if (user_data_->contains(phase)) { auto& phase_data = user_data_->at(phase); if (phase_data.find(object) != phase_data.end()) { return phase_data.at(object); @@ -132,8 +131,8 @@ ElmUserDataType RawData::getUserData(ElementIDStruct object, PhaseOffset offset) CommMapType RawData::getComm(PhaseOffset offset) const { auto phase = getNumCompletedPhases() + offset.phases; - if (auto it = proc_comm_->find(phase); it != proc_comm_->end()) { - return it->second; + if (auto it = proc_comm_->find(phase); it != nullptr) { + return *it; } else { return CommMapType{}; } diff --git a/src/vt/vrt/collection/balance/model/raw_data.h b/src/vt/vrt/collection/balance/model/raw_data.h index 2eeb65d6a3..9d186c3168 100644 --- a/src/vt/vrt/collection/balance/model/raw_data.h +++ b/src/vt/vrt/collection/balance/model/raw_data.h @@ -66,9 +66,9 @@ struct RawData : public LoadModel { ElmUserDataType getUserData(ElementIDStruct object, PhaseOffset when) const override; CommMapType getComm(PhaseOffset when) const override; - void setLoads(std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data) override; + void setLoads(LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data) override; ObjectIterator begin() const override; @@ -78,9 +78,9 @@ struct RawData : public LoadModel { unsigned int getNumPastPhasesNeeded(unsigned int look_back) const override; // Observer pointers to the underlying data. In operation, these would be owned by NodeLBData - std::unordered_map const* proc_load_; - std::unordered_map const* proc_comm_; - std::unordered_map const* user_data_; + LoadMapBufferType const* proc_load_; + CommMapBufferType const* proc_comm_; + DataMapBufferType const* user_data_; PhaseType last_completed_phase_ = ~0; }; // class RawData diff --git a/src/vt/vrt/collection/balance/model/weighted_messages.h b/src/vt/vrt/collection/balance/model/weighted_messages.h index 937c537a2c..d787f5ef4b 100644 --- a/src/vt/vrt/collection/balance/model/weighted_messages.h +++ b/src/vt/vrt/collection/balance/model/weighted_messages.h @@ -65,9 +65,9 @@ struct WeightedMessages : public ComposedModel { per_byte_weight_(in_per_byte_weight) { } void setLoads( - std::unordered_map const* proc_load, - std::unordered_map const* proc_comm, - std::unordered_map const* user_data + LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const* user_data ) override { proc_comm_ = proc_comm; ComposedModel::setLoads(proc_load, proc_comm, user_data); @@ -77,7 +77,7 @@ struct WeightedMessages : public ComposedModel { private: // observer pointer to the underlying comm data - std::unordered_map const* proc_comm_; + CommMapBufferType const* proc_comm_; LoadType per_msg_weight_; LoadType per_byte_weight_; diff --git a/src/vt/vrt/collection/balance/node_lb_data.cc b/src/vt/vrt/collection/balance/node_lb_data.cc index 75935ac138..8a69373817 100644 --- a/src/vt/vrt/collection/balance/node_lb_data.cc +++ b/src/vt/vrt/collection/balance/node_lb_data.cc @@ -90,26 +90,27 @@ bool NodeLBData::migrateObjTo(ElementIDStruct obj_id, NodeType to_node) { return true; } -std::unordered_map const* +LoadMapBufferType const* NodeLBData::getNodeLoad() const { return &lb_data_->node_data_; } -std::unordered_map const* +DataMapBufferType const* NodeLBData::getUserData() const { return &lb_data_->user_defined_lb_info_; } -std::unordered_map const* +DataMapBufferType const* NodeLBData::getPhaseAttributes() const { return &lb_data_->node_user_attributes_; } -std::unordered_map const* NodeLBData::getNodeComm() const { +CommMapBufferType const* +NodeLBData::getNodeComm() const { return &lb_data_->node_comm_; } -std::unordered_map> const* NodeLBData::getNodeSubphaseComm() const { +util::container::CircularPhasesBuffer> const* NodeLBData::getNodeSubphaseComm() const { return &lb_data_->node_subphase_comm_; } @@ -118,8 +119,7 @@ ElmUserDataType const* NodeLBData::getNodeAttributes() const { } CommMapType* NodeLBData::getNodeComm(PhaseType phase) { - auto iter = lb_data_->node_comm_.find(phase); - return (iter != lb_data_->node_comm_.end()) ? &iter->second : nullptr; + return lb_data_->node_comm_.find(phase); } void NodeLBData::clearLBData() { @@ -128,27 +128,30 @@ void NodeLBData::clearLBData() { next_elm_ = 1; } -void NodeLBData::startIterCleanup(PhaseType phase, unsigned int look_back) { - if (phase >= look_back) { - lb_data_->node_data_.erase(phase - look_back); - lb_data_->node_comm_.erase(phase - look_back); - lb_data_->node_subphase_comm_.erase(phase - look_back); - lb_data_->user_defined_lb_info_.erase(phase - look_back); - lb_data_->node_user_attributes_.erase(phase - look_back); - } - +void NodeLBData::startIterCleanup() { // Clear migrate lambdas and proxy lookup since LB is complete NodeLBData::node_migrate_.clear(); node_collection_lookup_.clear(); node_objgroup_lookup_.clear(); } +void NodeLBData::resizeLBDataHistory(uint32_t new_hist_len) { + hist_lb_data_size_ = new_hist_len; + + if (lb_data_) { + lb_data_->resizeHistory(hist_lb_data_size_); + } + + startIterCleanup(); +} + ElementIDType NodeLBData::getNextElm() { return next_elm_++; } void NodeLBData::initialize() { lb_data_ = std::make_unique(); + lb_data_->resizeHistory(hist_lb_data_size_); #if vt_check_enabled(lblite) if (theConfig()->vt_lb_data) { @@ -328,6 +331,9 @@ void NodeLBData::addNodeLBData( "NodeLBData::addNodeLBData: id={}\n", id ); + // Ensure that buffers can hold request amount of data + in->setHistoryCapacity(hist_lb_data_size_); + auto const phase = in->getPhase(); auto const& total_load = in->getLoad(phase, focused_subphase); @@ -374,9 +380,6 @@ void NodeLBData::addNodeLBData( } in->updatePhase(1); - - auto model = theLBManager()->getLoadModel(); - in->releaseLBDataFromUnneededPhases(phase, model->getNumPastPhasesNeeded()); } VirtualProxyType NodeLBData::getCollectionProxyForElement( diff --git a/src/vt/vrt/collection/balance/node_lb_data.h b/src/vt/vrt/collection/balance/node_lb_data.h index 5088090e3d..c39753d0cb 100644 --- a/src/vt/vrt/collection/balance/node_lb_data.h +++ b/src/vt/vrt/collection/balance/node_lb_data.h @@ -57,6 +57,7 @@ #include "vt/vrt/collection/balance/lb_data_holder.h" #include "vt/vrt/collection/types/storage/storable.h" #include "vt/utils/file_spec/spec.h" +#include "vt/utils/container/circular_phases_buffer.h" #include #include @@ -146,7 +147,7 @@ struct NodeLBData : runtime::component::Component { /** * \internal \brief Cleanup after LB runs */ - void startIterCleanup(PhaseType phase, unsigned int look_back); + void startIterCleanup(); /** * \internal \brief Load and broadcast the LB specification file @@ -172,28 +173,28 @@ struct NodeLBData : runtime::component::Component { * * \return an observer pointer to the load map */ - std::unordered_map const* getNodeLoad() const; + LoadMapBufferType const* getNodeLoad() const; /** * \internal \brief Get stored object comm graph * * \return an observer pointer to the comm graph */ - std::unordered_map const* getNodeComm() const; + CommMapBufferType const* getNodeComm() const; /** * \internal \brief Get the user-defined LB data * * \return an observer pointer to the user-defined LB data */ - std::unordered_map const* getUserData() const; + DataMapBufferType const* getUserData() const; /** * \internal \brief Get the user-defined attributes * * \return an observer pointer to the user-defined attributes */ - std::unordered_map const* getPhaseAttributes() const; + DataMapBufferType const* getPhaseAttributes() const; /** * \internal \brief Get stored object comm data for a specific phase @@ -209,7 +210,7 @@ struct NodeLBData : runtime::component::Component { * * \return an observer pointer to the comm subphase graph */ - std::unordered_map> const* getNodeSubphaseComm() const; + util::container::CircularPhasesBuffer> const* getNodeSubphaseComm() const; /** * \internal \brief Get stored node attributes @@ -270,6 +271,13 @@ struct NodeLBData : runtime::component::Component { */ LBDataHolder* getLBData() { return lb_data_.get(); } + /** + * \brief Set the amount of historical LB data which should be retained + * + * \param[in] new_hist_len the amount of LB data to retain + */ + void resizeLBDataHistory(uint32_t new_hist_len); + template void serialize(SerializerT& s) { s | proxy_ @@ -280,7 +288,8 @@ struct NodeLBData : runtime::component::Component { | next_elm_ | created_dir_ | lb_data_writer_ - | lb_data_; + | lb_data_ + | hist_lb_data_size_; } private: @@ -313,6 +322,8 @@ struct NodeLBData : runtime::component::Component { std::unique_ptr lb_data_writer_ = nullptr; /// The struct that holds all the LB data std::unique_ptr lb_data_ = nullptr; + //// The amount of historical LB data to hold + uint32_t hist_lb_data_size_ = 0; }; }}}} /* end namespace vt::vrt::collection::balance */ diff --git a/tests/unit/collection/test_collection_manager.cc b/tests/unit/collection/test_collection_manager.cc index d086a079d3..0183e1be42 100644 --- a/tests/unit/collection/test_collection_manager.cc +++ b/tests/unit/collection/test_collection_manager.cc @@ -52,7 +52,9 @@ namespace vt { namespace tests { namespace unit { -struct TestCollectionManager : TestParallelHarness {}; +namespace TestCollectionManager { + +struct TestCollectionManager : TestParallelHarness { }; struct TestCol : vt::Collection { static void colHandler(TestCol*) {} @@ -89,4 +91,4 @@ TEST_F(TestCollectionManager, test_collection_manager_proxy_deletion) { } } -}}} // end namespace vt::tests::unit +}}}} // end namespace vt::tests::unit::TestCollectionManager diff --git a/tests/unit/collection/test_lb.extended.cc b/tests/unit/collection/test_lb.extended.cc index 8995b4b179..5548b797e3 100644 --- a/tests/unit/collection/test_lb.extended.cc +++ b/tests/unit/collection/test_lb.extended.cc @@ -553,6 +553,7 @@ TEST_F(TestRestoreLBData, test_restore_lb_data_data_1) { }); vt::vrt::collection::balance::LBDataHolder lbdh; + lbdh.resizeHistory(num_phases); PhaseType write_phase = 0; using CommKey = vt::elm::CommKey; @@ -618,8 +619,8 @@ TEST_F(TestRestoreLBData, test_restore_lb_data_data_1) { // compare the whole-phase load data in detail for (auto &phase_data : lbdh.node_data_) { auto phase = phase_data.first; - EXPECT_FALSE(lbdh_read.node_data_.find(phase) == lbdh_read.node_data_.end()); - if (lbdh_read.node_data_.find(phase) == lbdh_read.node_data_.end()) { + EXPECT_FALSE(!lbdh_read.node_data_.contains(phase)); + if (!lbdh_read.node_data_.contains(phase)) { fmt::print( "Phase {} in whole-phase loads were not read in", phase @@ -751,6 +752,7 @@ TEST_P(TestDumpUserdefinedData, test_dump_userdefined_json) { }); vt::vrt::collection::balance::LBDataHolder lbdh; + lbdh.resizeHistory(num_phases); PhaseType write_phase = 0; { @@ -814,6 +816,7 @@ TEST_P(TestDumpAttributesFieldData, test_dump_attributes_json) { }); vt::vrt::collection::balance::LBDataHolder lbdh; + lbdh.resizeHistory(num_phases); PhaseType phase = 0; { diff --git a/tests/unit/collection/test_lb_data_holder.cc b/tests/unit/collection/test_lb_data_holder.cc index 09cc2ba174..df8bf9248f 100644 --- a/tests/unit/collection/test_lb_data_holder.cc +++ b/tests/unit/collection/test_lb_data_holder.cc @@ -75,6 +75,7 @@ void addPhasesDataToJson(nlohmann::json& json, PhaseType amountOfPhasesToAdd, st } LBDataHolder dh; + dh.resizeHistory(amountOfPhasesToAdd); for (unsigned i = 0; i < amountOfPhasesToAdd; i++) { for (auto&& elm : ids[i]) { dh.node_data_[i][elm] = LoadSummary{3.}; @@ -370,7 +371,7 @@ TEST_F(TestLBDataHolder, test_lb_entity_attributes) { auto id = vt::vrt::collection::balance::ElementIDStruct{524291, 0}; LBDataHolder testObj(json); - EXPECT_TRUE(testObj.node_user_attributes_.find(0) != testObj.node_user_attributes_.end()); + EXPECT_TRUE(testObj.node_user_attributes_.contains(0)); EXPECT_TRUE(testObj.node_user_attributes_[0].find(id) != testObj.node_user_attributes_[0].end()); auto attributes = testObj.node_user_attributes_[0][id]; EXPECT_EQ(123, std::get(attributes["intSample"])); diff --git a/tests/unit/collection/test_lb_data_retention.cc b/tests/unit/collection/test_lb_data_retention.cc index fea6645da4..c4c1af908b 100644 --- a/tests/unit/collection/test_lb_data_retention.cc +++ b/tests/unit/collection/test_lb_data_retention.cc @@ -58,13 +58,44 @@ namespace vt { namespace tests { namespace unit { +namespace TestLBDataRetention { + +void validatePersistedPhases(std::vector expected_phases) { + #if vt_check_enabled(lblite) + // Check maps size + EXPECT_EQ(expected_phases.size(), theNodeLBData()->getLBData()->node_comm_.size()); + EXPECT_EQ(expected_phases.size(), theNodeLBData()->getLBData()->node_data_.size()); + EXPECT_EQ(expected_phases.size(), theNodeLBData()->getLBData()->node_subphase_comm_.size()); + EXPECT_EQ(expected_phases.size(), theNodeLBData()->getLBData()->user_defined_json_.size()); + EXPECT_EQ(expected_phases.size(), theNodeLBData()->getLBData()->user_defined_lb_info_.size()); + EXPECT_EQ(expected_phases.size(), theNodeLBData()->getLBData()->node_user_attributes_.size()); + // Check if each phase is present + for(auto&& phase : expected_phases) { + EXPECT_TRUE(theNodeLBData()->getLBData()->node_comm_.contains(phase)); + EXPECT_TRUE(theNodeLBData()->getLBData()->node_data_.contains(phase)); + EXPECT_TRUE(theNodeLBData()->getLBData()->node_subphase_comm_.contains(phase)); + EXPECT_TRUE(theNodeLBData()->getLBData()->user_defined_json_.contains(phase)); + EXPECT_TRUE(theNodeLBData()->getLBData()->user_defined_lb_info_.contains(phase)); + EXPECT_TRUE(theNodeLBData()->getLBData()->node_user_attributes_.contains(phase)); + } + #else + (void)expected_phases; + EXPECT_EQ(0, theNodeLBData()->getLBData()->node_comm_.size()); + EXPECT_EQ(0, theNodeLBData()->getLBData()->node_data_.size()); + EXPECT_EQ(0, theNodeLBData()->getLBData()->node_subphase_comm_.size()); + EXPECT_EQ(0, theNodeLBData()->getLBData()->user_defined_json_.size()); + EXPECT_EQ(0, theNodeLBData()->getLBData()->user_defined_lb_info_.size()); + EXPECT_EQ(0, theNodeLBData()->getLBData()->node_user_attributes_.size()); + #endif +} + struct TestCol : vt::Collection { - unsigned int prev_calls_ = thePhase()->getCurrentPhase(); - unsigned int prevCalls() { return prev_calls_++; } + static void insertValue(TestCol* col) { + col->valInsert("foo", 10, true, true, true); + } static void colHandler(TestCol* col) { - auto& lb_data = col->lb_data_; auto load_phase_count = lb_data.getLoadPhaseCount(); auto comm_phase_count = lb_data.getCommPhaseCount(); @@ -72,21 +103,19 @@ struct TestCol : vt::Collection { auto sp_comm_phase_count = lb_data.getSubphaseCommPhaseCount(); #if vt_check_enabled(lblite) - auto phase = col->prevCalls(); + auto phase = thePhase()->getCurrentPhase(); auto model = theLBManager()->getLoadModel(); - auto phases_needed = model->getNumPastPhasesNeeded(); - if (phase > phases_needed) { - // updatePhase will have caused entries to be added for the - // next phase already - EXPECT_EQ(load_phase_count, phases_needed + 1); - EXPECT_EQ(sp_load_phase_count, phases_needed + 1); - EXPECT_EQ(comm_phase_count, phases_needed + 1); - EXPECT_EQ(sp_comm_phase_count, phases_needed + 1); + auto buffers_size = std::max(model->getNumPastPhasesNeeded(), theConfig()->vt_lb_data_retention); + if (phase >= buffers_size) { + EXPECT_EQ(load_phase_count, buffers_size); + EXPECT_EQ(sp_load_phase_count, buffers_size); + EXPECT_EQ(comm_phase_count, buffers_size); + EXPECT_EQ(sp_comm_phase_count, buffers_size); } else if (phase == 0) { - EXPECT_EQ(load_phase_count, phase); - EXPECT_EQ(sp_load_phase_count, phase); - EXPECT_EQ(comm_phase_count, phase); - EXPECT_EQ(sp_comm_phase_count, phase); + EXPECT_EQ(load_phase_count, std::size_t{1}); + EXPECT_EQ(sp_load_phase_count, std::size_t{1}); + EXPECT_EQ(comm_phase_count, std::size_t{0}); + EXPECT_EQ(sp_comm_phase_count, std::size_t{0}); } else { // updatePhase will have caused entries to be added for the // next phase already @@ -96,10 +125,30 @@ struct TestCol : vt::Collection { EXPECT_EQ(sp_comm_phase_count, phase + 1); } #else - EXPECT_EQ(load_phase_count, 0); - EXPECT_EQ(sp_load_phase_count, 0); - EXPECT_EQ(comm_phase_count, 0); - EXPECT_EQ(sp_comm_phase_count, 0); + EXPECT_EQ(load_phase_count, std::size_t{0}); + EXPECT_EQ(sp_load_phase_count, std::size_t{0}); + EXPECT_EQ(comm_phase_count, std::size_t{0}); + EXPECT_EQ(sp_comm_phase_count, std::size_t{0}); + #endif + } + + static void emptyDataHandler(TestCol* col) { + auto& lb_data = col->lb_data_; + auto load_phase_count = lb_data.getLoadPhaseCount(); + auto comm_phase_count = lb_data.getCommPhaseCount(); + auto sp_load_phase_count = lb_data.getSubphaseLoadPhaseCount(); + auto sp_comm_phase_count = lb_data.getSubphaseCommPhaseCount(); + + #if vt_check_enabled(lblite) + EXPECT_EQ(load_phase_count, std::size_t{1}); + EXPECT_EQ(sp_load_phase_count, std::size_t{1}); + EXPECT_EQ(comm_phase_count, std::size_t{1}); + EXPECT_EQ(sp_comm_phase_count, std::size_t{1}); + #else + EXPECT_EQ(load_phase_count, std::size_t{0}); + EXPECT_EQ(sp_load_phase_count, std::size_t{0}); + EXPECT_EQ(comm_phase_count, std::size_t{0}); + EXPECT_EQ(sp_comm_phase_count, std::size_t{0}); #endif } }; @@ -130,6 +179,7 @@ TEST_F(TestLBDataRetention, test_lbdata_retention_last1) { proxy = vt::theCollection()->constructCollective( range, "test_lbstats_retention_last1" ); + proxy.broadcastCollective(); }); // Get the base model, assert it's valid @@ -150,10 +200,15 @@ TEST_F(TestLBDataRetention, test_lbdata_retention_last1) { // Go to the next phase. vt::thePhase()->nextPhaseCollective(); } + + // Check the phases persisted in the node + validatePersistedPhases({4}); } TEST_F(TestLBDataRetention, test_lbdata_retention_last2) { static constexpr int const num_phases = 6; + // Minimum retention lower than the amount of phases needed by model - will be ignored + theConfig()->vt_lb_data_retention = 1; auto range = vt::Index1D(num_elms); @@ -164,6 +219,7 @@ TEST_F(TestLBDataRetention, test_lbdata_retention_last2) { proxy = vt::theCollection()->constructCollective( range, "test_lbstats_retention_last2" ); + proxy.broadcastCollective(); }); // Get the base model, assert it's valid @@ -184,10 +240,15 @@ TEST_F(TestLBDataRetention, test_lbdata_retention_last2) { // Go to the next phase. vt::thePhase()->nextPhaseCollective(); } + + // Check the phases persisted in the node + validatePersistedPhases({4,5}); } TEST_F(TestLBDataRetention, test_lbdata_retention_last4) { static constexpr int const num_phases = 8; + // Minimum retention equal to the amount of phases needed by model + theConfig()->vt_lb_data_retention = 4; auto range = vt::Index1D(num_elms); @@ -198,6 +259,7 @@ TEST_F(TestLBDataRetention, test_lbdata_retention_last4) { proxy = vt::theCollection()->constructCollective( range, "test_lbstats_retention_last4" ); + proxy.broadcastCollective(); }); // Get the base model, assert it's valid @@ -218,6 +280,289 @@ TEST_F(TestLBDataRetention, test_lbdata_retention_last4) { // Go to the next phase. vt::thePhase()->nextPhaseCollective(); } + + // Check the phases persisted in the node + validatePersistedPhases({4,5,6,7}); +} + +TEST_F(TestLBDataRetention, test_lbdata_config_retention_higher) { + // Minimum retention higher than the amount of phases needed by model + theConfig()->vt_lb_data_retention = 6; + + // We must have more or equal number of elements than nodes for this test to + // work properly + EXPECT_GE(num_elms, vt::theContext()->getNumNodes()); + + auto range = vt::Index1D(num_elms); + + vt::vrt::collection::CollectionProxy proxy; + + // Construct two collections + runInEpochCollective([&]{ + proxy = vt::theCollection()->constructCollective( + range, "test_lbdata_config_retention_higher" + ); + proxy.broadcastCollective(); + }); + + // Get the base model, assert it's valid + auto base = theLBManager()->getBaseLoadModel(); + EXPECT_NE(base, nullptr); + + // Create a new model + auto persist = std::make_shared(base, 4U); + + // Set the new model + theLBManager()->setLoadModel(persist); + + for (uint32_t i=0; ivt_lb_data_retention * 2; ++i) { + runInEpochCollective([&]{ + // Do some work. + proxy.broadcastCollective(); + }); + // Go to the next phase. + vt::thePhase()->nextPhaseCollective(); + } + + // Check the phases persisted in the node + validatePersistedPhases({6,7,8,9,10,11}); +} + +TEST_F(TestLBDataRetention, test_lbdata_retention_model_switch_1) { + static constexpr int const first_stage_num_phases = 11; + theConfig()->vt_lb_data_retention = 0; + + // We must have more or equal number of elements than nodes for this test to + // work properly + EXPECT_GE(num_elms, vt::theContext()->getNumNodes()); + auto range = vt::Index1D(num_elms); + vt::vrt::collection::CollectionProxy proxy; + + // Construct two collections + runInEpochCollective([&]{ + proxy = vt::theCollection()->constructCollective( + range, "test_lbdata_retention_model_switch_1" + ); + proxy.broadcastCollective(); + }); + + // Get the base model, assert it's valid + auto base = theLBManager()->getBaseLoadModel(); + EXPECT_NE(base, nullptr); + + // Create a new models + auto model_10_phases = std::make_shared(base, 10U); + auto model_1_phase = std::make_shared(base, 1U); + + // Set model which needs 10 phases of data + theLBManager()->setLoadModel(model_10_phases); + + for (uint32_t i=0; i(); + }); + // Go to the next phase. + vt::thePhase()->nextPhaseCollective(); + } + + // Set model which needs only 1 phase of data + theLBManager()->setLoadModel(model_1_phase); + + // Check the phases persisted in the node + validatePersistedPhases({10}); + + // Go to next phase to update size of the buffers in TestCol. + vt::thePhase()->nextPhaseCollective(); + + for (uint32_t i=0; i(); + proxy.broadcastCollective(); + }); + // Go to the next phase. + vt::thePhase()->nextPhaseCollective(); + } + + // Check the phases persisted in the node + validatePersistedPhases({22}); +} + +TEST_F(TestLBDataRetention, test_lbdata_retention_model_switch_2) { + static constexpr int const first_stage_num_phases = 6; + theConfig()->vt_lb_data_retention = 0; + + // We must have more or equal number of elements than nodes for this test to + // work properly + EXPECT_GE(num_elms, vt::theContext()->getNumNodes()); + auto range = vt::Index1D(num_elms); + vt::vrt::collection::CollectionProxy proxy; + + // Construct two collections + runInEpochCollective([&]{ + proxy = vt::theCollection()->constructCollective( + range, "test_lbdata_retention_model_switch_2" + ); + proxy.broadcastCollective(); + }); + + // Get the base model, assert it's valid + auto base = theLBManager()->getBaseLoadModel(); + EXPECT_NE(base, nullptr); + + // Create and set model which needs 10 phases of data + auto model_10_phases = std::make_shared(base, 10U); + theLBManager()->setLoadModel(model_10_phases); + + // Do only 6 phases of work + for (uint32_t i=0; i(); + }); + // Go to the next phase. + vt::thePhase()->nextPhaseCollective(); + } + + // Check the phases persisted in the node + validatePersistedPhases({0,1,2,3,4,5}); + + // Set model which needs only 1 phase of data + auto model_1_phase = std::make_shared(base, 1U); + theLBManager()->setLoadModel(model_1_phase); + + // Check the phases persisted in the node + validatePersistedPhases({5}); + + // Go to next phase to update size of the buffers in TestCol. + vt::thePhase()->nextPhaseCollective(); + + // Do another 10 phases of work + for (uint32_t i=0; i<10; ++i) { + runInEpochCollective([&]{ + // Do some work. + proxy.broadcastCollective(); + proxy.broadcastCollective(); + }); + // Go to the next phase. + vt::thePhase()->nextPhaseCollective(); + } + + // Check the phases persisted in the node + validatePersistedPhases({16}); +} + +struct TestLBDataRetentionOnCheckpoint : TestParallelHarness { + virtual void SetUp() override { + TestParallelHarness::SetUp(); + + // We must have more or equal number of elements than nodes for this test to + // work properly + SET_MAX_NUM_NODES_CONSTRAINT(num_elms); + } +}; + +TEST_F(TestLBDataRetentionOnCheckpoint, test_lbdata_retention_checkpoint) { + static constexpr int const num_phases = 8; + std::string const checkpoint_name(getUniqueFilenameWithRanks()); + auto range = vt::Index1D(num_elms); + + { + theConfig()->vt_lb_data_retention = 4; + + auto this_node = theContext()->getNode(); + vt::vrt::collection::CollectionProxy proxy; + + runInEpochCollective([&]{ + proxy = vt::theCollection()->constructCollective( + range, "test_lbdata_retention_checkpoint" + ); + proxy.broadcastCollective(); + }); + + // Get the base model, assert it's valid + auto base = theLBManager()->getBaseLoadModel(); + EXPECT_NE(base, nullptr); + + // Create a new model + auto persist = std::make_shared(base, 4U); + + // Set the new model + theLBManager()->setLoadModel(persist); + + for (int i=0; i(); + }); + // Go to the next phase. + vt::thePhase()->nextPhaseCollective(); + } + + // Check the phases persisted in the node + validatePersistedPhases({4,5,6,7}); + + vt::runInEpochCollective([&]{ + vt_print(gen, "checkpointToFile\n"); + vt::theCollection()->checkpointToFile(proxy, checkpoint_name); + }); + + vt::runInEpochCollective([&]{ + if (this_node == 0) { + proxy.destroy(); + } + }); + } + + // Destroy and init fresh instance + TestLBDataRetentionOnCheckpoint::destroyVt(); + vt_print(gen, "newVtInstance\n"); + TestLBDataRetentionOnCheckpoint::initVt(); + + auto proxy_new = vt::theCollection()->constructCollective( + range, "test_lbdata_retention_checkpoint" + ); + + // Get the base model, assert it's valid + auto base = theLBManager()->getBaseLoadModel(); + EXPECT_NE(base, nullptr); + + // Create a new model + auto persist = std::make_shared(base, 4U); + + // Set the new model + theLBManager()->setLoadModel(persist); + + vt::runInEpochCollective([&]{ + // Now, restore from the previous distribution + vt_print(gen, "restoreFromFileInPlace\n"); + vt::theCollection()->restoreFromFileInPlace( + proxy_new, range, checkpoint_name + ); + }); + + // After restore node LB data is empty + validatePersistedPhases({}); + + runInEpochCollective([&]{ + proxy_new.broadcastCollective(); + }); + vt::thePhase()->nextPhaseCollective(); + + validatePersistedPhases({0}); + + // Do more work. + for (int i=0; i(); + }); + // Go to the next phase. + vt::thePhase()->nextPhaseCollective(); + } + + // Check the phases persisted in the node + validatePersistedPhases({5, 6, 7, 8}); } -}}} // end namespace vt::tests::unit +}}}} // end namespace vt::tests::unit::TestLBDataRetention diff --git a/tests/unit/collection/test_model_comm_overhead.nompi.cc b/tests/unit/collection/test_model_comm_overhead.nompi.cc index 20d3bfc44e..3e1f338796 100644 --- a/tests/unit/collection/test_model_comm_overhead.nompi.cc +++ b/tests/unit/collection/test_model_comm_overhead.nompi.cc @@ -68,11 +68,9 @@ using vt::vrt::collection::balance::PhaseOffset; using vt::vrt::collection::balance::SubphaseLoadMapType; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; - -using ProcLoadMap = std::unordered_map; -using ProcSubphaseLoadMap = std::unordered_map; -using ProcCommMap = std::unordered_map; -using UserDataMap = std::unordered_map; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; static auto num_phases = 0; @@ -82,9 +80,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - ProcLoadMap const* proc_load, - ProcCommMap const*, - UserDataMap const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -111,7 +109,7 @@ struct StubModel : LoadModel { unsigned int getNumPastPhasesNeeded(unsigned int look_back = 0) const override { return look_back; } private: - ProcLoadMap const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestModelCommOverhead, test_model_comm_overhead_1) { @@ -126,29 +124,24 @@ TEST_F(TestModelCommOverhead, test_model_comm_overhead_1) { // Element 3 (home node == 3) ElementIDStruct const elem3 = {3, 3}; - ProcLoadMap proc_load = {{0, LoadMapType{{elem2, {LoadType{150}, {}}}}}}; - - ProcCommMap proc_comm = { - {0, - CommMapType{// Node 1 -> Node 2 - {{CommKeyType::CollectionTag{}, elem1, elem2, false}, - CommVolume{20.0, 2}}, - - // Node 3 -> Node 2 - {{CommKeyType::CollectionTag{}, elem3, elem2, false}, - CommVolume{5.0, 5}}} - }, - {1, - CommMapType{ - // Node 3 -> Node 2 - {{CommKeyType::CollectionTag{}, elem3, elem2, false}, - CommVolume{500.0, 50}}, - - // Node 1 -> Node 2 - {{CommKeyType::CollectionTag{}, elem1, elem2, false}, - CommVolume{25.0, 10}}} - } - }; + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{{elem2, {LoadType{150}, {}}}}; + + CommMapBufferType proc_comm(2); + proc_comm[0] = CommMapType{ + // Node 1 -> Node 2 + {{CommKeyType::CollectionTag{}, elem1, elem2, false}, CommVolume{20.0, 2}}, + + // Node 3 -> Node 2 + {{CommKeyType::CollectionTag{}, elem3, elem2, false}, CommVolume{5.0, 5}}}; + proc_comm[1] = + CommMapType{// Node 3 -> Node 2 + {{CommKeyType::CollectionTag{}, elem3, elem2, false}, + CommVolume{500.0, 50}}, + + // Node 1 -> Node 2 + {{CommKeyType::CollectionTag{}, elem1, elem2, false}, + CommVolume{25.0, 10}}}; constexpr auto per_msg_weight = 3.0; constexpr auto per_byte_weight = 5.0; diff --git a/tests/unit/collection/test_model_linear_model.nompi.cc b/tests/unit/collection/test_model_linear_model.nompi.cc index 09c37adfbf..a6152f4720 100644 --- a/tests/unit/collection/test_model_linear_model.nompi.cc +++ b/tests/unit/collection/test_model_linear_model.nompi.cc @@ -66,6 +66,9 @@ using vt::vrt::collection::balance::PhaseOffset; using vt::vrt::collection::balance::SubphaseLoadMapType; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; struct StubModel : LoadModel { @@ -73,9 +76,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - std::unordered_map const* proc_load, - std::unordered_map const*, - std::unordered_map const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -95,7 +98,7 @@ struct StubModel : LoadModel { unsigned int getNumPastPhasesNeeded(unsigned int look_back = 0) const override { return look_back; } private: - std::unordered_map const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestLinearModel, test_model_linear_model_1) { @@ -107,10 +110,10 @@ TEST_F(TestLinearModel, test_model_linear_model_1) { // For linear regression there needs to be at least 2 phases completed // so we begin with 1 phase already done - std::unordered_map proc_loads{{0, LoadMapType{ + LoadMapBufferType proc_loads(num_test_interations + 1); + proc_loads[0] = LoadMapType{ {ElementIDStruct{1,this_node}, {LoadType{10}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{40}, {}}} - }}}; + {ElementIDStruct{2,this_node}, {LoadType{40}, {}}}}; test_model->setLoads(&proc_loads, nullptr, nullptr); test_model->updateLoads(0); diff --git a/tests/unit/collection/test_model_multiple_phases.nompi.cc b/tests/unit/collection/test_model_multiple_phases.nompi.cc index 793e89099a..e9e280a87c 100644 --- a/tests/unit/collection/test_model_multiple_phases.nompi.cc +++ b/tests/unit/collection/test_model_multiple_phases.nompi.cc @@ -64,6 +64,9 @@ using vt::vrt::collection::balance::CommMapType; using vt::vrt::collection::balance::ObjectIterator; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; struct StubModel : LoadModel { @@ -71,9 +74,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - std::unordered_map const* proc_load, - std::unordered_map const*, - std::unordered_map const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -95,24 +98,24 @@ struct StubModel : LoadModel { unsigned int getNumPastPhasesNeeded(unsigned int look_back = 0) const override { return look_back; } private: - std::unordered_map const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestModelMultiplePhases, test_model_multiple_phases_1) { NodeType this_node = 0; - std::unordered_map proc_loads = { - {0, LoadMapType{ + LoadMapBufferType proc_loads(4); + proc_loads[0] = LoadMapType{ {ElementIDStruct{1,this_node}, {LoadType{10}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{40}, {}}}}}, - {1, LoadMapType{ + {ElementIDStruct{2,this_node}, {LoadType{40}, {}}}}; + proc_loads[1] = LoadMapType{ {ElementIDStruct{1,this_node}, {LoadType{20}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{30}, {}}}}}, - {2, LoadMapType{ + {ElementIDStruct{2,this_node}, {LoadType{30}, {}}}}; + proc_loads[2] = LoadMapType{ {ElementIDStruct{1,this_node}, {LoadType{30}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{10}, {}}}}}, - {3, LoadMapType{ + {ElementIDStruct{2,this_node}, {LoadType{10}, {}}}}; + proc_loads[3] = LoadMapType{ {ElementIDStruct{1,this_node}, {LoadType{40}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{5}, {}}}}}}; + {ElementIDStruct{2,this_node}, {LoadType{5}, {}}}}; auto test_model = std::make_shared(std::make_shared(), 4); diff --git a/tests/unit/collection/test_model_naive_persistence.nompi.cc b/tests/unit/collection/test_model_naive_persistence.nompi.cc index 50e6d7933f..e29fd903af 100644 --- a/tests/unit/collection/test_model_naive_persistence.nompi.cc +++ b/tests/unit/collection/test_model_naive_persistence.nompi.cc @@ -64,6 +64,9 @@ using vt::vrt::collection::balance::CommMapType; using vt::vrt::collection::balance::ObjectIterator; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; static int32_t getIndexFromPhase(int32_t phase) { return std::max(0, -1 * phase - 1); @@ -75,9 +78,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - std::unordered_map const* proc_load, - std::unordered_map const*, - std::unordered_map const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -98,24 +101,24 @@ struct StubModel : LoadModel { unsigned int getNumPastPhasesNeeded(unsigned int look_back = 0) const override { return look_back; } private: - std::unordered_map const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestModelNaivePersistence, test_model_naive_persistence_1) { NodeType this_node = 0; - std::unordered_map proc_loads = { - {0, LoadMapType{ - {ElementIDStruct{1,this_node}, {LoadType{10}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{40}, {}}}}}, - {1, LoadMapType{ - {ElementIDStruct{1,this_node}, {LoadType{4}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{10}, {}}}}}, - {2, LoadMapType{ - {ElementIDStruct{1,this_node}, {LoadType{20}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{50}, {}}}}}, - {3, LoadMapType{ - {ElementIDStruct{1,this_node}, {LoadType{40}, {}}}, - {ElementIDStruct{2,this_node}, {LoadType{100}, {}}}}}}; + LoadMapBufferType proc_loads(4); + proc_loads[0] = LoadMapType{ + {ElementIDStruct{1, this_node}, {LoadType{10}, {}}}, + {ElementIDStruct{2, this_node}, {LoadType{40}, {}}}}; + proc_loads[1] = LoadMapType{ + {ElementIDStruct{1, this_node}, {LoadType{4}, {}}}, + {ElementIDStruct{2, this_node}, {LoadType{10}, {}}}}; + proc_loads[2] = LoadMapType{ + {ElementIDStruct{1, this_node}, {LoadType{20}, {}}}, + {ElementIDStruct{2, this_node}, {LoadType{50}, {}}}}; + proc_loads[3] = LoadMapType{ + {ElementIDStruct{1, this_node}, {LoadType{40}, {}}}, + {ElementIDStruct{2, this_node}, {LoadType{100}, {}}}}; auto test_model = std::make_shared(std::make_shared()); diff --git a/tests/unit/collection/test_model_norm.nompi.cc b/tests/unit/collection/test_model_norm.nompi.cc index 5d8a223ac3..21a99ac910 100644 --- a/tests/unit/collection/test_model_norm.nompi.cc +++ b/tests/unit/collection/test_model_norm.nompi.cc @@ -65,11 +65,9 @@ using vt::vrt::collection::balance::PhaseOffset; using vt::vrt::collection::balance::SubphaseLoadMapType; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; - -using ProcLoadMap = std::unordered_map; -using ProcSubphaseLoadMap = std::unordered_map; -using ProcCommMap = std::unordered_map; -using UserDataMap = std::unordered_map; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; constexpr auto num_subphases = 3; @@ -79,9 +77,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - ProcLoadMap const* proc_load, - ProcCommMap const*, - UserDataMap const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -102,16 +100,17 @@ struct StubModel : LoadModel { unsigned int getNumPastPhasesNeeded(unsigned int look_back = 0) const override { return look_back; } private: - ProcLoadMap const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestModelNorm, test_model_norm_1) { NodeType this_node = 0; - ProcLoadMap proc_load = { - {0, - LoadMapType{ - {ElementIDStruct{1,this_node}, {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, - {ElementIDStruct{2,this_node}, {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}}}; + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{ + {ElementIDStruct{1, this_node}, + {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, + {ElementIDStruct{2, this_node}, + {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}; auto test_model = std::make_shared(std::make_shared(), 3.0); test_model->setLoads(&proc_load, nullptr, nullptr); @@ -138,11 +137,12 @@ TEST_F(TestModelNorm, test_model_norm_1) { TEST_F(TestModelNorm, test_model_norm_2) { NodeType this_node = 0; - ProcLoadMap proc_load = { - {0, - LoadMapType{ - {ElementIDStruct{1,this_node}, {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, - {ElementIDStruct{2,this_node}, {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}}}; + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{ + {ElementIDStruct{1, this_node}, + {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, + {ElementIDStruct{2, this_node}, + {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}; // finite 'power' value auto test_model = std::make_shared(std::make_shared(), 3.0); @@ -167,11 +167,10 @@ TEST_F(TestModelNorm, test_model_norm_2) { TEST_F(TestModelNorm, test_model_norm_3) { NodeType this_node = 0; - ProcLoadMap proc_load = { - {0, - LoadMapType{ + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{ {ElementIDStruct{1,this_node}, {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, - {ElementIDStruct{2,this_node}, {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}}}; + {ElementIDStruct{2,this_node}, {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}; // infinite 'power' value auto test_model = std::make_shared( diff --git a/tests/unit/collection/test_model_persistence_median_last_n.nompi.cc b/tests/unit/collection/test_model_persistence_median_last_n.nompi.cc index 0107c4d251..fe1eeb90fe 100644 --- a/tests/unit/collection/test_model_persistence_median_last_n.nompi.cc +++ b/tests/unit/collection/test_model_persistence_median_last_n.nompi.cc @@ -66,6 +66,9 @@ using vt::vrt::collection::balance::CommMapType; using vt::vrt::collection::balance::ObjectIterator; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; struct StubModel : LoadModel { @@ -73,9 +76,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - std::unordered_map const* proc_load, - std::unordered_map const*, - std::unordered_map const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -95,7 +98,7 @@ struct StubModel : LoadModel { unsigned int getNumPastPhasesNeeded(unsigned int look_back = 0) const override { return look_back; } private: - std::unordered_map const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestModelPersistenceMedianLastN, test_model_persistence_median_last_n_1) { @@ -105,7 +108,7 @@ TEST_F(TestModelPersistenceMedianLastN, test_model_persistence_median_last_n_1) auto test_model = std::make_shared(std::make_shared(), 4); - std::unordered_map proc_loads(num_total_phases); + LoadMapBufferType proc_loads(num_total_phases); test_model->setLoads(&proc_loads, nullptr, nullptr); diff --git a/tests/unit/collection/test_model_raw_data.nompi.cc b/tests/unit/collection/test_model_raw_data.nompi.cc index 3dfca34863..63dd8fdfb0 100644 --- a/tests/unit/collection/test_model_raw_data.nompi.cc +++ b/tests/unit/collection/test_model_raw_data.nompi.cc @@ -64,13 +64,15 @@ using vt::vrt::collection::balance::PhaseOffset; using vt::vrt::collection::balance::SubphaseLoadMapType; using vt::vrt::collection::balance::ElmUserDataType; using vt::vrt::collection::balance::DataMapType; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; TEST_F(TestRawData, test_model_raw_data_scalar) { NodeType this_node = 0; auto test_model = std::make_shared(); - std::unordered_map proc_loads; + LoadMapBufferType proc_loads; test_model->setLoads(&proc_loads, nullptr, nullptr); EXPECT_TRUE(test_model->hasRawLoad()); EXPECT_FALSE(test_model->hasUserData()); // because passed a nullptr @@ -127,8 +129,8 @@ TEST_F(TestRawData, test_model_raw_user_data) { auto test_model = std::make_shared(); - std::unordered_map proc_loads; - std::unordered_map user_data; + LoadMapBufferType proc_loads; + DataMapBufferType user_data; test_model->setLoads(&proc_loads, nullptr, &user_data); EXPECT_TRUE(test_model->hasRawLoad()); EXPECT_TRUE(test_model->hasUserData()); diff --git a/tests/unit/collection/test_model_select_subphases.nompi.cc b/tests/unit/collection/test_model_select_subphases.nompi.cc index fb41feacbe..57648bcca1 100644 --- a/tests/unit/collection/test_model_select_subphases.nompi.cc +++ b/tests/unit/collection/test_model_select_subphases.nompi.cc @@ -64,11 +64,9 @@ using vt::vrt::collection::balance::SelectSubphases; using vt::vrt::collection::balance::SubphaseLoadMapType; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; - -using ProcLoadMap = std::unordered_map; -using ProcSubphaseLoadMap = std::unordered_map; -using ProcCommMap = std::unordered_map; -using UserDataMap = std::unordered_map; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; constexpr auto num_subphases = 3; @@ -78,9 +76,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - ProcLoadMap const* proc_load, - ProcCommMap const*, - UserDataMap const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -107,7 +105,7 @@ struct StubModel : LoadModel { } private: - ProcLoadMap const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestModelSelectSubphases, test_model_select_subphases_1) { @@ -115,11 +113,10 @@ TEST_F(TestModelSelectSubphases, test_model_select_subphases_1) { ElementIDStruct id1{1,this_node}; ElementIDStruct id2{2,this_node}; - ProcLoadMap proc_load = { - {0, - LoadMapType{ + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{ {id1, {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, - {id2, {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}}}; + {id2, {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}; std::vector subphases{2, 0, 1}; auto test_model = @@ -159,16 +156,12 @@ TEST_F(TestModelSelectSubphases, test_model_select_subphases_1) { TEST_F(TestModelSelectSubphases, test_model_select_subphases_2) { NodeType this_node = 0; - ProcLoadMap proc_load = { - {0, - LoadMapType{ - {ElementIDStruct{1,this_node}, - {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, - {ElementIDStruct{2,this_node}, - {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}} - } - } - }; + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{ + {ElementIDStruct{1, this_node}, + {LoadType{60}, {LoadType{10}, LoadType{20}, LoadType{30}}}}, + {ElementIDStruct{2, this_node}, + {LoadType{150}, {LoadType{40}, LoadType{50}, LoadType{60}}}}}; std::vector subphases{2, 1}; auto test_model = diff --git a/tests/unit/collection/test_model_weighted_communication_volume.nompi.cc b/tests/unit/collection/test_model_weighted_communication_volume.nompi.cc index a7cdc11ff3..f11c197ff2 100644 --- a/tests/unit/collection/test_model_weighted_communication_volume.nompi.cc +++ b/tests/unit/collection/test_model_weighted_communication_volume.nompi.cc @@ -68,10 +68,9 @@ using vt::vrt::collection::balance::ObjectIterator; using vt::vrt::collection::balance::PhaseOffset; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; - -using ProcLoadMap = std::unordered_map; -using ProcCommMap = std::unordered_map; -using UserDataMap = std::unordered_map; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; static auto num_phases = 0; @@ -80,9 +79,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - ProcLoadMap const* proc_load, - ProcCommMap const* proc_comm, - UserDataMap const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const* proc_comm, + DataMapBufferType const*) override { proc_load_ = proc_load; proc_comm_ = proc_comm; } @@ -114,8 +113,8 @@ struct StubModel : LoadModel { } private: - ProcLoadMap const* proc_load_ = nullptr; - ProcCommMap const* proc_comm_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; + CommMapBufferType const* proc_comm_ = nullptr; }; TEST_F(TestModelWeightedCommunicationVolume, test_model) { @@ -129,26 +128,22 @@ TEST_F(TestModelWeightedCommunicationVolume, test_model) { // Element 3 (home node == 3) ElementIDStruct const elem3 = {3, 3}; - ProcLoadMap proc_load = {{0, LoadMapType{{elem2, {LoadType{150}, {}}}}}}; - - ProcCommMap proc_comm = { - {0, - CommMapType{// Node 1 -> Node 2 - {{CommKeyType::CollectionTag{}, elem1, elem2, false}, - CommVolume{20.0, 2}}, - // Node 3 -> Node 2 - {{CommKeyType::CollectionTag{}, elem3, elem2, false}, - CommVolume{5.0, 5}}} - }, - {1, - CommMapType{// Node 3 -> Node 2 - {{CommKeyType::CollectionTag{}, elem3, elem2, false}, - CommVolume{500.0, 50}}, - // Node 1 -> Node 2 - {{CommKeyType::CollectionTag{}, elem1, elem2, false}, - CommVolume{25.0, 10}}} - } - }; + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{{elem2, {LoadType{150}, {}}}}; + + CommMapBufferType proc_comm(2); + proc_comm[0] = CommMapType{ + // Node 1 -> Node 2 + {{CommKeyType::CollectionTag{}, elem1, elem2, false}, CommVolume{20.0, 2}}, + // Node 3 -> Node 2 + {{CommKeyType::CollectionTag{}, elem3, elem2, false}, CommVolume{5.0, 5}}}; + proc_comm[1] = + CommMapType{// Node 3 -> Node 2 + {{CommKeyType::CollectionTag{}, elem3, elem2, false}, + CommVolume{500.0, 50}}, + // Node 1 -> Node 2 + {{CommKeyType::CollectionTag{}, elem1, elem2, false}, + CommVolume{25.0, 10}}}; constexpr auto alpha = 0.8; constexpr auto beta = 0.2; diff --git a/tests/unit/collection/test_model_weighted_messages.nompi.cc b/tests/unit/collection/test_model_weighted_messages.nompi.cc index d08f345d0e..63cc7bb2d6 100644 --- a/tests/unit/collection/test_model_weighted_messages.nompi.cc +++ b/tests/unit/collection/test_model_weighted_messages.nompi.cc @@ -66,10 +66,9 @@ using vt::vrt::collection::balance::ObjectIterator; using vt::vrt::collection::balance::PhaseOffset; using vt::vrt::collection::balance::LoadMapObjectIterator; using vt::vrt::collection::balance::DataMapType; - -using ProcLoadMap = std::unordered_map; -using ProcCommMap = std::unordered_map; -using UserDataMap = std::unordered_map; +using vt::vrt::collection::balance::LoadMapBufferType; +using vt::vrt::collection::balance::CommMapBufferType; +using vt::vrt::collection::balance::DataMapBufferType; static auto num_phases = 0; @@ -78,9 +77,9 @@ struct StubModel : LoadModel { virtual ~StubModel() = default; void setLoads( - ProcLoadMap const* proc_load, - ProcCommMap const*, - UserDataMap const*) override { + LoadMapBufferType const* proc_load, + CommMapBufferType const*, + DataMapBufferType const*) override { proc_load_ = proc_load; } @@ -111,7 +110,7 @@ struct StubModel : LoadModel { } private: - ProcLoadMap const* proc_load_ = nullptr; + LoadMapBufferType const* proc_load_ = nullptr; }; TEST_F(TestModelWeightedMessages, test_model) { @@ -125,26 +124,22 @@ TEST_F(TestModelWeightedMessages, test_model) { // Element 3 (home node == 3) ElementIDStruct const elem3 = {3, 3}; - ProcLoadMap proc_load = {{0, LoadMapType{{elem2, {LoadType{150}, {}}}}}}; - - ProcCommMap proc_comm = { - {0, - CommMapType{// Node 1 -> Node 2 - {{CommKeyType::CollectionTag{}, elem1, elem2, false}, - CommVolume{20.0, 2}}, - // Node 3 -> Node 2 - {{CommKeyType::CollectionTag{}, elem3, elem2, false}, - CommVolume{5.0, 5}}} - }, - {1, - CommMapType{// Node 3 -> Node 2 - {{CommKeyType::CollectionTag{}, elem3, elem2, false}, - CommVolume{500.0, 50}}, - // Node 1 -> Node 2 - {{CommKeyType::CollectionTag{}, elem1, elem2, false}, - CommVolume{25.0, 10}}} - } - }; + LoadMapBufferType proc_load(1); + proc_load[0] = LoadMapType{{elem2, {LoadType{150}, {}}}}; + + CommMapBufferType proc_comm(2); + proc_comm[0] = CommMapType{ + // Node 1 -> Node 2 + {{CommKeyType::CollectionTag{}, elem1, elem2, false}, CommVolume{20.0, 2}}, + // Node 3 -> Node 2 + {{CommKeyType::CollectionTag{}, elem3, elem2, false}, CommVolume{5.0, 5}}}; + proc_comm[1] = + CommMapType{// Node 3 -> Node 2 + {{CommKeyType::CollectionTag{}, elem3, elem2, false}, + CommVolume{500.0, 50}}, + // Node 1 -> Node 2 + {{CommKeyType::CollectionTag{}, elem1, elem2, false}, + CommVolume{25.0, 10}}}; constexpr auto per_msg_weight = 3.0; constexpr auto per_byte_weight = 5.0; diff --git a/tests/unit/collection/test_workload_data_migrator.cc b/tests/unit/collection/test_workload_data_migrator.cc index f034b30909..f6e7c1f752 100644 --- a/tests/unit/collection/test_workload_data_migrator.cc +++ b/tests/unit/collection/test_workload_data_migrator.cc @@ -94,6 +94,7 @@ setupWorkloads(PhaseType phase, size_t numElements) { } auto lbdh = std::make_shared(); + lbdh->resizeHistory(1); for (auto&& elmID : myElemList) { double tval = elmID.id * 2; @@ -791,6 +792,7 @@ setupManyWorkloads( } auto lbdh = std::make_shared(); + lbdh->resizeHistory(num_phases); PhaseType stop_phase = initial_phase + num_phases; for (PhaseType phase = initial_phase; phase < stop_phase; ++phase) { @@ -805,6 +807,7 @@ setupManyWorkloads( } auto scrambled_lbdh = std::make_shared(); + scrambled_lbdh->resizeHistory(num_phases); for (PhaseType phase = initial_phase; phase < stop_phase; ++phase) { auto base_load_model = setupBaseModel(phase, lbdh); diff --git a/tests/unit/lb/test_offlinelb.cc b/tests/unit/lb/test_offlinelb.cc index b470e61f1c..a5c72349c3 100644 --- a/tests/unit/lb/test_offlinelb.cc +++ b/tests/unit/lb/test_offlinelb.cc @@ -126,6 +126,7 @@ TEST_F(TestOfflineLB, test_offlinelb_1) { } LBDataHolder dh; + dh.resizeHistory(num_phases); for (PhaseType i = 0; i < num_phases; i++) { for (auto&& elm : ids[i]) { dh.node_data_[i][elm] = LoadSummary{3}; @@ -197,6 +198,7 @@ TEST_F(TestOfflineLB, test_offlinelb_2) { } LBDataHolder dh; + dh.resizeHistory(num_phases); for (PhaseType i = 0; i < num_phases; i++) { if (i != 1 and i != 2 and i != 5 and i != 8 and i != 9) { auto& elms = ids[i]; diff --git a/tests/unit/runtime/test_initialization.cc b/tests/unit/runtime/test_initialization.cc index 132321fe2e..2e0e04e7a1 100644 --- a/tests/unit/runtime/test_initialization.cc +++ b/tests/unit/runtime/test_initialization.cc @@ -544,6 +544,7 @@ void prepareLBDataFiles(const std::string file_name_without_ext) { } LBDataHolder dh; + dh.resizeHistory(num_phases); for (PhaseType i = 0; i < num_phases; i++) { for (auto&& elm : ids[i]) { dh.node_data_[i][elm] = LoadSummary{3}; diff --git a/tests/unit/test_parallel_harness.h b/tests/unit/test_parallel_harness.h index d5c8e1c7fc..a96260f72e 100644 --- a/tests/unit/test_parallel_harness.h +++ b/tests/unit/test_parallel_harness.h @@ -61,8 +61,6 @@ extern char** test_argv; template struct TestParallelHarnessAny : TestHarnessAny { virtual void SetUp() override { - using namespace vt; - TestHarnessAny::SetUp(); #if vt_feature_cmake_test_trace_on @@ -84,15 +82,24 @@ struct TestParallelHarnessAny : TestHarnessAny { if (!init) { MPI_Init(&test_argc, &test_argv); } - MPI_Comm comm = MPI_COMM_WORLD; - auto const new_args = injectAdditionalArgs(test_argc, test_argv); - auto custom_argc = new_args.first; - auto custom_argv = new_args.second; + + // inject args only once + injectAdditionalArgs(test_argc, test_argv); + + initVt(); + } + + void initVt() { + using namespace vt; + + int custom_argc = additional_args_.size() - 1; + char** custom_argv = additional_args_.data(); vtAssert( custom_argv[custom_argc] == nullptr, "The value of argv[argc] should always be 0" ); // communicator is duplicated. + MPI_Comm comm = MPI_COMM_WORLD; CollectiveOps::initialize(custom_argc, custom_argv, true, &comm); #if DEBUG_TEST_HARNESS_PRINT @@ -103,6 +110,12 @@ struct TestParallelHarnessAny : TestHarnessAny { } virtual void TearDown() override { + destroyVt(); + + TestHarnessAny::TearDown(); + } + + void destroyVt() { using namespace vt; try { @@ -117,8 +130,6 @@ struct TestParallelHarnessAny : TestHarnessAny { #endif CollectiveOps::finalize(); - - TestHarnessAny::TearDown(); } protected: @@ -134,8 +145,7 @@ struct TestParallelHarnessAny : TestHarnessAny { } private: - std::pair - injectAdditionalArgs(int old_argc, char** old_argv) { + void injectAdditionalArgs(int old_argc, char** old_argv) { additional_args_.insert( additional_args_.begin(), old_argv, old_argv + old_argc ); @@ -143,10 +153,6 @@ struct TestParallelHarnessAny : TestHarnessAny { addAdditionalArgs(); additional_args_.emplace_back(nullptr); - int custom_argc = additional_args_.size() - 1; - char** custom_argv = additional_args_.data(); - - return std::make_pair(custom_argc, custom_argv); } /** diff --git a/tests/unit/utils/test_circular_phases_buffer.nompi.cc b/tests/unit/utils/test_circular_phases_buffer.nompi.cc new file mode 100644 index 0000000000..386ffa5a1f --- /dev/null +++ b/tests/unit/utils/test_circular_phases_buffer.nompi.cc @@ -0,0 +1,298 @@ +/* +//@HEADER +// ***************************************************************************** +// +// test_circular_phases_buffer.nompi.cc +// DARMA/vt => Virtual Transport +// +// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#include + +#include +#include "test_harness.h" + +namespace vt { namespace tests { namespace unit { + +using TestCircularPhasesBuffer = TestHarness; + +struct Dummy { + PhaseType x; +}; + +using CircularBufferType = util::container::CircularPhasesBuffer; + +void validatePresentPhases(CircularBufferType& buffer, std::vector expected) { + for (auto&& phase : expected) { + EXPECT_TRUE(buffer.contains(phase)) << " Phase: " << phase; + EXPECT_EQ(phase, buffer.at(phase).x) << " Phase: " << phase; + EXPECT_EQ(phase, buffer[phase].x) << " Phase: " << phase; + EXPECT_EQ(phase, buffer.find(phase)->x) << " Phase: " << phase; + } +} + +void validateMissingPhases(CircularBufferType& buffer, std::vector expected) { + for (auto&& phase : expected) { + EXPECT_FALSE(buffer.contains(phase)) << " Phase: " << phase; + EXPECT_EQ(nullptr, buffer.find(phase)) << " Phase: " << phase; + } +} + +TEST_F(TestCircularPhasesBuffer, test_circular_phases_buffer_empty) { + CircularBufferType buffer; + + EXPECT_FALSE(buffer.contains(0)); + EXPECT_EQ(std::size_t{0}, buffer.size()); + EXPECT_TRUE(buffer.empty()); + EXPECT_EQ(std::numeric_limits::max(), buffer.frontPhase()); + EXPECT_EQ(nullptr, buffer.find(0)); + + buffer.resize(2); + + EXPECT_FALSE(buffer.contains(0)); + EXPECT_EQ(std::size_t{0}, buffer.size()); + EXPECT_TRUE(buffer.empty()); + EXPECT_EQ(std::numeric_limits::max(), buffer.frontPhase()); + EXPECT_EQ(nullptr, buffer.find(0)); + + buffer.clear(); + + EXPECT_FALSE(buffer.contains(1)); + EXPECT_EQ(std::size_t{0}, buffer.size()); + EXPECT_TRUE(buffer.empty()); + EXPECT_EQ(std::numeric_limits::max(), buffer.frontPhase()); + EXPECT_EQ(nullptr, buffer.find(0)); + + buffer.resize(4); + buffer[0] = {0}; + + EXPECT_TRUE(buffer.contains(0)); + EXPECT_EQ(std::size_t{1}, buffer.size()); + EXPECT_FALSE(buffer.empty()); + EXPECT_EQ(PhaseType{0}, buffer.frontPhase()); + EXPECT_EQ(PhaseType{0}, buffer.frontData().x); + EXPECT_NE(nullptr, buffer.find(0)); + + buffer[1] = {1}; + buffer[2] = {4}; + buffer[3] = {6}; + buffer[4] = {8}; + + EXPECT_TRUE(buffer.contains(1)); + EXPECT_EQ(std::size_t{4}, buffer.size()); + EXPECT_FALSE(buffer.empty()); + EXPECT_EQ(PhaseType{4}, buffer.frontPhase()); + EXPECT_EQ(PhaseType{8}, buffer.frontData().x); + EXPECT_EQ(nullptr, buffer.find(0)); + EXPECT_NE(nullptr, buffer.find(3)); + + buffer.clear(); + + EXPECT_FALSE(buffer.contains(0)); + EXPECT_EQ(std::size_t{0}, buffer.size()); + EXPECT_TRUE(buffer.empty()); + EXPECT_EQ(std::numeric_limits::max(), buffer.frontPhase()); + EXPECT_EQ(nullptr, buffer.find(0)); +} + +TEST_F(TestCircularPhasesBuffer, test_circular_phases_buffer_single_element) { + CircularBufferType buffer{1}; + + EXPECT_EQ(false, buffer.contains(0)); + + for (PhaseType i = 0; i < 5; i++) { + buffer.store(i, {i}); + EXPECT_EQ(i == 0, buffer.contains(0)); + EXPECT_EQ(i == 1, buffer.contains(1)); + EXPECT_EQ(i == 2, buffer.contains(2)); + EXPECT_EQ(i == 3, buffer.contains(3)); + EXPECT_EQ(i == 4, buffer.contains(4)); + EXPECT_EQ(i == 5, buffer.contains(5)); + EXPECT_EQ(i == 6, buffer.contains(6)); + + EXPECT_EQ(i, buffer.frontPhase()); + EXPECT_EQ(i, buffer.frontData().x); + EXPECT_NE(nullptr, buffer.find(i)); + EXPECT_NO_THROW(buffer.at(i)); + } +} + +TEST_F(TestCircularPhasesBuffer, test_circular_phases_buffer_multi_elements) { + CircularBufferType buffer{5}; + + for (PhaseType i = 0; i < 15; i++) { + buffer.store(i, {i}); + + EXPECT_EQ(i, buffer.frontPhase()); + EXPECT_EQ(i, buffer.frontData().x); + EXPECT_NE(nullptr, buffer.find(i)); + EXPECT_NO_THROW(buffer.at(i)); + } + std::vector finalOutput = {10, 11, 12, 13, 14}; + validatePresentPhases(buffer, finalOutput); +} + +TEST_F(TestCircularPhasesBuffer, test_circular_phases_buffer_store) { + CircularBufferType buffer; + + buffer.store(2, {2}); + validatePresentPhases(buffer, {2}); + validateMissingPhases(buffer, {0, 1}); + + buffer.resize(10); + + validatePresentPhases(buffer, {2}); + validateMissingPhases(buffer, {0, 1, 3, 4, 5, 6, 7, 8, 9}); + + // store series of elements + for (PhaseType i = 3; i < 15; i++) { + buffer.store(i, {i}); + } + + validateMissingPhases(buffer, {0, 1, 2, 3, 4}); + std::vector finalOutput = {5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; + validatePresentPhases(buffer, finalOutput); +} + +TEST_F(TestCircularPhasesBuffer, test_circular_phases_buffer_resize) { + CircularBufferType buffer{10}; + + for (PhaseType i = 0; i <= 15; i++) { + buffer[i] = {i}; + } + validatePresentPhases(buffer, {6, 7, 8, 9, 10, 11, 12, 13, 14, 15}); + + buffer.resize(5); + validatePresentPhases(buffer, {11, 12, 13, 14, 15}); + validateMissingPhases(buffer, {6, 7, 8, 9, 10}); + + for (PhaseType i = 15; i <= 32; i++) { + buffer[i] = {i}; + } + validatePresentPhases(buffer, {28, 29, 30, 31, 32}); + validateMissingPhases(buffer, {11, 12, 13, 14, 15}); + + buffer.resize(9); + + validatePresentPhases(buffer, {28, 29, 30, 31, 32}); + + for (PhaseType i = 33; i <= 35; i++) { + buffer[i] = {i}; + } + validatePresentPhases(buffer, {28, 29, 30, 31, 32, 33, 34, 35}); + + buffer.resize(1); + validatePresentPhases(buffer, {35}); + validateMissingPhases(buffer, {28, 29, 30, 31, 32, 33, 34}); +} + +TEST_F(TestCircularPhasesBuffer, test_circular_phases_buffer_restart_from) { + CircularBufferType buffer{1}; + + buffer.restartFrom(5); + EXPECT_EQ(PhaseType{5}, buffer.frontPhase()); + + buffer[5] = {5}; + EXPECT_EQ(PhaseType{5}, buffer.frontPhase()); + EXPECT_EQ(PhaseType{5}, buffer.frontData().x); + + buffer.restartFrom(10); + EXPECT_EQ(PhaseType{10}, buffer.frontPhase()); + EXPECT_EQ(PhaseType{5}, buffer.frontData().x); + + buffer.restartFrom(0); + EXPECT_EQ(PhaseType{0}, buffer.frontPhase()); + EXPECT_EQ(PhaseType{5}, buffer.frontData().x); +} + +TEST_F(TestCircularPhasesBuffer, test_circular_phases_buffer_forward_iter) { + CircularBufferType buffer; + + for (auto& ele: buffer) { + EXPECT_EQ(true, false) << " Phase: " << ele.first; + } + for (auto&& ele : buffer) { + EXPECT_EQ(true, false) << " Phase: " << ele.first; + } + + buffer.resize(9); + buffer[0] = {0}; + buffer[3] = {3}; + buffer[7] = {7}; + + { + std::vector visited; + for (auto&& ele : buffer) { + visited.push_back(ele.first); + } + validatePresentPhases(buffer, visited); + EXPECT_EQ(visited.size(), buffer.size()); + } + + { + std::vector visited; + for (auto& ele : buffer) { + visited.push_back(ele.first); + } + validatePresentPhases(buffer, visited); + EXPECT_EQ(visited.size(), buffer.size()); + } + + for (PhaseType i = 0; i <= 15; i++) { + buffer[i] = {i}; + } + + { + std::vector visited; + for (auto&& ele : buffer) { + visited.push_back(ele.first); + } + validatePresentPhases(buffer, visited); + EXPECT_EQ(visited.size(), buffer.size()); + } + + { + std::vector visited; + for (auto& ele : buffer) { + visited.push_back(ele.first); + } + validatePresentPhases(buffer, visited); + EXPECT_EQ(visited.size(), buffer.size()); + } +} + +}}} /* end namespace vt::tests::unit */