Skip to content

Commit

Permalink
#1554: DR: use location manager properly to fix race condition
Browse files Browse the repository at this point in the history
  • Loading branch information
lifflander committed Oct 4, 2021
1 parent 37b16ad commit e4d7a36
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 49 deletions.
52 changes: 14 additions & 38 deletions src/vt/datarep/dr.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,16 @@

namespace vt { namespace datarep {

namespace detail {

template <typename T>
struct DataResponseMsg;

template <typename T>
struct DataRequestMsg;

} /* end namespace detail */

struct DataReplicator : runtime::component::Component<DataReplicator> {

std::string name() override { return "DataReplicator"; }
Expand All @@ -82,55 +92,21 @@ struct DataReplicator : runtime::component::Component<DataReplicator> {
template <typename T>
void unregisterHandle(DataRepIDType handle_id);

template <typename T>
struct DataRequestMsg : vt::Message {
using MessageParentType = vt::Message;
vt_msg_serialize_prohibited();

DataRequestMsg(NodeType in_requestor, DataRepIDType in_handle_id)
: requestor_(in_requestor),
handle_id_(in_handle_id)
{ }

NodeType requestor_ = uninitialized_destination;
DataRepIDType handle_id_ = no_datarep;
};

template <typename T>
struct DataResponseMsg : vt::Message {
using MessageParentType = vt::Message;
vt_msg_serialize_if_needed_by_parent_or_type1(T);

DataResponseMsg() = default; // for serializer
DataResponseMsg(DataRepIDType in_handle_id, T const& data)
: handle_id_(in_handle_id),
data_(std::make_unique<T>(data))
{ }

template <typename SerializerT>
void serialize(SerializerT& s) {
MessageParentType::serialize(s);
s | handle_id_;
s | data_;
}

DataRepIDType handle_id_ = no_datarep;
std::unique_ptr<T> data_;
};

template <typename T>
bool requestData(DataRepIDType handle_id, bool* ready_ptr);

template <typename T>
T const& getDataRef(DataRepIDType handle_id) const;

private:
template <typename T>
static void staticRequestHandler(detail::DataRequestMsg<T>* msg);

template <typename T>
void dataIncomingHandler(DataResponseMsg<T>* msg);
void dataIncomingHandler(detail::DataResponseMsg<T>* msg);

template <typename T>
void dataRequestHandler(DataRequestMsg<T>* msg);
void dataRequestHandler(detail::DataRequestMsg<T>* msg);

NodeType getHomeNode(DataRepIDType handle_id) const {
return handle_id >> 48;
Expand Down
33 changes: 22 additions & 11 deletions src/vt/datarep/dr.impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@

#include "vt/topos/location/manager.h"
#include "vt/datarep/dr.h"
#include "vt/datarep/msg.h"
#include "vt/objgroup/manager.h"

namespace vt { namespace datarep {
Expand Down Expand Up @@ -118,19 +119,29 @@ bool DataReplicator::requestData(DataRepIDType handle_id, bool* ready_ptr) {
"requestData: handle_id={} remote request\n", handle_id
);
waiting_[handle_id].push_back(ready_ptr);
theLocMan()->dataRep->getLocation(
handle_id, getHomeNode(handle_id), [=](NodeType loc) {
auto const this_node = theContext()->getNode();
auto proxy = objgroup::proxy::Proxy<DataReplicator>{proxy_};
proxy[loc].send<
DataRequestMsg<T>, &DataReplicator::dataRequestHandler<T>
>(this_node, handle_id);
}

using MsgType = detail::DataRequestMsg<T>;
auto const this_node = theContext()->getNode();
auto msg = makeMessage<MsgType>(this_node, handle_id);
theLocMan()->dataRep->routeMsgHandler<MsgType, staticRequestHandler<T>>(
handle_id, getHomeNode(handle_id), msg.get()
);
return false;
}
}

template <typename T>
/*static*/ void DataReplicator::staticRequestHandler(
detail::DataRequestMsg<T>* msg
) {
auto proxy = objgroup::proxy::Proxy<DataReplicator>{theDR()->proxy_};
auto loc = theContext()->getNode();
proxy[loc].invoke<
decltype(&DataReplicator::dataRequestHandler<T>),
&DataReplicator::dataRequestHandler<T>
>(msg);
}

template <typename T>
T const& DataReplicator::getDataRef(DataRepIDType handle_id) const {
auto iter = local_store_.find(handle_id);
Expand All @@ -143,7 +154,7 @@ T const& DataReplicator::getDataRef(DataRepIDType handle_id) const {
}

template <typename T>
void DataReplicator::dataIncomingHandler(DataResponseMsg<T>* msg) {
void DataReplicator::dataIncomingHandler(detail::DataResponseMsg<T>* msg) {
auto const han_id = msg->handle_id_;
vt_debug_print(
normal, gen,
Expand All @@ -167,7 +178,7 @@ void DataReplicator::dataIncomingHandler(DataResponseMsg<T>* msg) {
}

template <typename T>
void DataReplicator::dataRequestHandler(DataRequestMsg<T>* msg) {
void DataReplicator::dataRequestHandler(detail::DataRequestMsg<T>* msg) {
auto const requestor = msg->requestor_;
auto const handle_id = msg->handle_id_;
auto const found = requestData<T>(handle_id, nullptr);
Expand All @@ -179,7 +190,7 @@ void DataReplicator::dataRequestHandler(DataRequestMsg<T>* msg) {
if (found) {
auto proxy = objgroup::proxy::Proxy<DataReplicator>{proxy_};
proxy[requestor].template send<
DataResponseMsg<T>, &DataReplicator::dataIncomingHandler<T>
detail::DataResponseMsg<T>, &DataReplicator::dataIncomingHandler<T>
>(handle_id, getDataRef<T>(handle_id));
}
}
Expand Down
90 changes: 90 additions & 0 deletions src/vt/datarep/msg.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
//@HEADER
// *****************************************************************************
//
// msg.h
// DARMA/vt => Virtual Transport
//
// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC
// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
// Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived from this
// software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact [email protected]
//
// *****************************************************************************
//@HEADER
*/

#if !defined INCLUDED_VT_DATAREP_MSG_H
#define INCLUDED_VT_DATAREP_MSG_H

#include "vt/topos/location/message/msg.h"

namespace vt { namespace datarep { namespace detail {

template <typename T>
struct DataRequestMsg : LocationRoutedMsg<DataRepIDType, vt::Message> {
using MessageParentType = vt::Message;
vt_msg_serialize_prohibited();

DataRequestMsg(NodeType in_requestor, DataRepIDType in_handle_id)
: requestor_(in_requestor),
handle_id_(in_handle_id)
{ }

NodeType requestor_ = uninitialized_destination;
DataRepIDType handle_id_ = no_datarep;
};

template <typename T>
struct DataResponseMsg : vt::Message {
using MessageParentType = vt::Message;
vt_msg_serialize_if_needed_by_parent_or_type1(T);

DataResponseMsg() = default; // for serializer
DataResponseMsg(DataRepIDType in_handle_id, T const& data)
: handle_id_(in_handle_id),
data_(std::make_unique<T>(data))
{ }

template <typename SerializerT>
void serialize(SerializerT& s) {
MessageParentType::serialize(s);
s | handle_id_;
s | data_;
}

DataRepIDType handle_id_ = no_datarep;
std::unique_ptr<T> data_;
};


}}} /* end namespace vt::datarep::detail */

#endif /*INCLUDED_VT_DATAREP_MSG_H*/

0 comments on commit e4d7a36

Please sign in to comment.