diff --git a/src/nrniv/have2want.cpp b/src/nrniv/have2want.cpp deleted file mode 100644 index 5ac55709aa..0000000000 --- a/src/nrniv/have2want.cpp +++ /dev/null @@ -1,259 +0,0 @@ -/* -To be included by a file that desires rendezvous rank exchange functionality. -Need to define HAVEWANT_t, HAVEWANT_alltoallv, and HAVEWANT2Int -The latter is a map or unordered_map. -E.g. std::unordered_map -*/ - -#ifdef have2want_cpp -#error "This implementation can only be included once" -// The static function names used to involve a macro name (NrnHash) but now, -// with the use of std::..., it may be the case this could be included -// multiple times or even transformed into a template. -#endif - -#define have2want_cpp - -/* - -A rank owns a set of HAVEWANT_t keys and wants information associated with -a set of HAVEWANT_t keys owned by unknown ranks. Owners do not know which -ranks want their information. Ranks that want info do not know which ranks -own that info. - -The have_to_want function returns two new vectors of keys along with -associated count and displacement vectors of length nhost and nhost+1 -respectively. Note that a send_to_want_displ[i+1] = - send_to_want_cnt[i] + send_to_want_displ[i] . - -send_to_want[send_to_want_displ[i] to send_to_want_displ[i+1]] contains -the keys from this rank for which rank i wants information. - -recv_from_have[recv_from_have_displ[i] to recv_from_have_displ[i+1] contains -the keys from which rank i is sending information to this rank. - -Note that on rank i, the order of keys in the rank j area of send_to_want -is the same order of keys on rank j in the ith area in recv_from_have. - -The rendezvous_rank function is used to parallelize this computation -and minimize memory usage so that no single rank ever needs to know all keys. -*/ - -#ifndef HAVEWANT_t -#define HAVEWANT_t int -#endif - -// round robin default rendezvous rank function -static int default_rendezvous(HAVEWANT_t key) { - return key % nrnmpi_numprocs; -} - -static int* cnt2displ(int* cnt) { - int* displ = new int[nrnmpi_numprocs + 1]; - displ[0] = 0; - for (int i = 0; i < nrnmpi_numprocs; ++i) { - displ[i + 1] = displ[i] + cnt[i]; - } - return displ; -} - -static int* srccnt2destcnt(int* srccnt) { - int* destcnt = new int[nrnmpi_numprocs]; - nrnmpi_int_alltoall(srccnt, destcnt, 1); - return destcnt; -} - -static void rendezvous_rank_get(HAVEWANT_t* data, - int size, - HAVEWANT_t*& sdata, - int*& scnt, - int*& sdispl, - HAVEWANT_t*& rdata, - int*& rcnt, - int*& rdispl, - int (*rendezvous_rank)(HAVEWANT_t)) { - int nhost = nrnmpi_numprocs; - - // count what gets sent - scnt = new int[nhost]; - for (int i = 0; i < nhost; ++i) { - scnt[i] = 0; - } - for (int i = 0; i < size; ++i) { - int r = (*rendezvous_rank)(data[i]); - ++scnt[r]; - } - - sdispl = cnt2displ(scnt); - rcnt = srccnt2destcnt(scnt); - rdispl = cnt2displ(rcnt); - sdata = new HAVEWANT_t[sdispl[nhost] + 1]; // ensure not 0 size - rdata = new HAVEWANT_t[rdispl[nhost] + 1]; // ensure not 0 size - // scatter data into sdata by recalculating scnt. - for (int i = 0; i < nhost; ++i) { - scnt[i] = 0; - } - for (int i = 0; i < size; ++i) { - int r = (*rendezvous_rank)(data[i]); - sdata[sdispl[r] + scnt[r]] = data[i]; - ++scnt[r]; - } - HAVEWANT_alltoallv(sdata, scnt, sdispl, rdata, rcnt, rdispl); -} - -static void have_to_want(HAVEWANT_t* have, - int have_size, - HAVEWANT_t* want, - int want_size, - HAVEWANT_t*& send_to_want, - int*& send_to_want_cnt, - int*& send_to_want_displ, - HAVEWANT_t*& recv_from_have, - int*& recv_from_have_cnt, - int*& recv_from_have_displ, - int (*rendezvous_rank)(HAVEWANT_t)) { - // 1) Send have and want to the rendezvous ranks. - // 2) Rendezvous rank matches have and want. - // 3) Rendezvous ranks tell the want ranks which ranks own the keys - // 4) Ranks that want tell owner ranks where to send. - - int nhost = nrnmpi_numprocs; - - // 1) Send have and want to the rendezvous ranks. - HAVEWANT_t *have_s_data, *have_r_data; - int *have_s_cnt, *have_s_displ, *have_r_cnt, *have_r_displ; - rendezvous_rank_get(have, - have_size, - have_s_data, - have_s_cnt, - have_s_displ, - have_r_data, - have_r_cnt, - have_r_displ, - rendezvous_rank); - delete[] have_s_cnt; - delete[] have_s_displ; - delete[] have_s_data; - // assume it is an error if two ranks have the same key so create - // hash table of key2rank. Will also need it for matching have and want - HAVEWANT2Int havekey2rank = HAVEWANT2Int(have_r_displ[nhost] + 1); // ensure not empty. - for (int r = 0; r < nhost; ++r) { - for (int i = 0; i < have_r_cnt[r]; ++i) { - HAVEWANT_t key = have_r_data[have_r_displ[r] + i]; - if (havekey2rank.find(key) != havekey2rank.end()) { - hoc_execerr_ext( - "internal error in have_to_want: key %lld owned by multiple ranks\n", - (long long) key); - } - havekey2rank[key] = r; - } - } - delete[] have_r_data; - delete[] have_r_cnt; - delete[] have_r_displ; - - HAVEWANT_t *want_s_data, *want_r_data; - int *want_s_cnt, *want_s_displ, *want_r_cnt, *want_r_displ; - rendezvous_rank_get(want, - want_size, - want_s_data, - want_s_cnt, - want_s_displ, - want_r_data, - want_r_cnt, - want_r_displ, - rendezvous_rank); - - // 2) Rendezvous rank matches have and want. - // we already have made the havekey2rank map. - // Create an array parallel to want_r_data which contains the ranks that - // have that data. - int n = want_r_displ[nhost]; - int* want_r_ownerranks = new int[n]; - for (int r = 0; r < nhost; ++r) { - for (int i = 0; i < want_r_cnt[r]; ++i) { - int ix = want_r_displ[r] + i; - HAVEWANT_t key = want_r_data[ix]; - auto search = havekey2rank.find(key); - if (search == havekey2rank.end()) { - hoc_execerr_ext( - "internal error in have_to_want: key = %lld is wanted but does not exist\n", - (long long) key); - } - want_r_ownerranks[ix] = search->second; - } - } - delete[] want_r_data; - - // 3) Rendezvous ranks tell the want ranks which ranks own the keys - // The ranks that want keys need to know the ranks that own those keys. - // The want_s_ownerranks will be parallel to the want_s_data. - // That is, each item defines the rank from which information associated - // with that key is coming from - int* want_s_ownerranks = new int[want_s_displ[nhost]]; - if (nrn_sparse_partrans > 0) { - nrnmpi_int_alltoallv_sparse(want_r_ownerranks, - want_r_cnt, - want_r_displ, - want_s_ownerranks, - want_s_cnt, - want_s_displ); - } else { - nrnmpi_int_alltoallv(want_r_ownerranks, - want_r_cnt, - want_r_displ, - want_s_ownerranks, - want_s_cnt, - want_s_displ); - } - - delete[] want_r_ownerranks; - delete[] want_r_cnt; - delete[] want_r_displ; - - // 4) Ranks that want tell owner ranks where to send. - // Finished with the rendezvous ranks. The ranks that want keys know the - // owner ranks for those keys. The next step is for the want ranks to - // tell the owner ranks where to send. - // The parallel want_s_ownerranks and want_s_data are now uselessly ordered - // by rendezvous rank. Reorganize so that want ranks can tell owner ranks - // what they want. - n = want_s_displ[nhost]; - delete[] want_s_displ; - for (int i = 0; i < nhost; ++i) { - want_s_cnt[i] = 0; - } - HAVEWANT_t* old_want_s_data = want_s_data; - want_s_data = new HAVEWANT_t[n]; - // compute the counts - for (int i = 0; i < n; ++i) { - int r = want_s_ownerranks[i]; - ++want_s_cnt[r]; - } - want_s_displ = cnt2displ(want_s_cnt); - for (int i = 0; i < nhost; ++i) { - want_s_cnt[i] = 0; - } // recount while filling - for (int i = 0; i < n; ++i) { - int r = want_s_ownerranks[i]; - HAVEWANT_t key = old_want_s_data[i]; - want_s_data[want_s_displ[r] + want_s_cnt[r]] = key; - ++want_s_cnt[r]; - } - delete[] want_s_ownerranks; - delete[] old_want_s_data; - want_r_cnt = srccnt2destcnt(want_s_cnt); - want_r_displ = cnt2displ(want_r_cnt); - want_r_data = new HAVEWANT_t[want_r_displ[nhost]]; - HAVEWANT_alltoallv( - want_s_data, want_s_cnt, want_s_displ, want_r_data, want_r_cnt, want_r_displ); - // now the want_r_data on the have_ranks are grouped according to the ranks - // that want those keys. - - send_to_want = want_r_data; - send_to_want_cnt = want_r_cnt; - send_to_want_displ = want_r_displ; - recv_from_have = want_s_data; - recv_from_have_cnt = want_s_cnt; - recv_from_have_displ = want_s_displ; -} diff --git a/src/nrniv/have2want.hpp b/src/nrniv/have2want.hpp new file mode 100644 index 0000000000..11b22d28ed --- /dev/null +++ b/src/nrniv/have2want.hpp @@ -0,0 +1,202 @@ +#pragma once + +#include +#include +#include + +/* +A rank owns a set of T keys and wants information associated with +a set of T keys owned by unknown ranks. Owners do not know which +ranks want their information. Ranks that want info do not know which ranks +own that info. + +The have_to_want function returns two new vectors of keys along with +associated count and displacement vectors of length nhost and nhost+1 +respectively. Note that a send_to_want_displ[i+1] = + send_to_want_cnt[i] + send_to_want_displ[i] . + +send_to_want[send_to_want_displ[i] to send_to_want_displ[i+1]] contains +the keys from this rank for which rank i wants information. + +recv_from_have[recv_from_have_displ[i] to recv_from_have_displ[i+1] contains +the keys from which rank i is sending information to this rank. + +Note that on rank i, the order of keys in the rank j area of send_to_want +is the same order of keys on rank j in the ith area in recv_from_have. + +The rendezvous_rank function is used to parallelize this computation +and minimize memory usage so that no single rank ever needs to know all keys. +*/ + +// round robin rendezvous rank function +template +int rendezvous_rank(const T& key) { + return key % nrnmpi_numprocs; +} + +template +struct Data { + std::vector data{}; + std::vector cnt{}; + std::vector displ{}; +}; + +static std::vector cnt2displ(const std::vector& cnt) { + std::vector displ(nrnmpi_numprocs + 1); + std::partial_sum(cnt.cbegin(), cnt.cend(), displ.begin() + 1); + return displ; +} + +static std::vector srccnt2destcnt(std::vector srccnt) { + std::vector destcnt(nrnmpi_numprocs); + nrnmpi_int_alltoall(srccnt.data(), destcnt.data(), 1); + return destcnt; +} + +template +static std::tuple, Data> rendezvous_rank_get(const std::vector& data, + F alltoall_function) { + int nhost = nrnmpi_numprocs; + + Data s; + // count what gets sent + s.cnt = std::vector(nhost); + + for (const auto& e: data) { + int r = rendezvous_rank(e); + s.cnt[r] += 1; + } + + s.displ = cnt2displ(s.cnt); + s.data.resize(s.displ[nhost] + 1); + + Data r; + r.cnt = srccnt2destcnt(s.cnt); + r.displ = cnt2displ(r.cnt); + r.data.resize(r.displ[nhost]); + // scatter data into sdata by recalculating s.cnt. + std::fill(s.cnt.begin(), s.cnt.end(), 0); + for (const auto& e: data) { + int rank = rendezvous_rank(e); + s.data[s.displ[rank] + s.cnt[rank]] = e; + s.cnt[rank] += 1; + } + alltoall_function(s, r); + return {s, r}; +} + +template +std::pair, Data> have_to_want(const std::vector& have, + const std::vector& want, + F alltoall_function) { + // 1) Send have and want to the rendezvous ranks. + // 2) Rendezvous rank matches have and want. + // 3) Rendezvous ranks tell the want ranks which ranks own the keys + // 4) Ranks that want tell owner ranks where to send. + + int nhost = nrnmpi_numprocs; + + // 1) Send have and want to the rendezvous ranks. + + // hash table of key2rank. Will also need it for matching have and want + std::unordered_map havekey2rank{}; + { + auto [_, have_r] = rendezvous_rank_get(have, alltoall_function); + // assume it is an error if two ranks have the same key so create + havekey2rank.reserve(have_r.displ[nhost] + 1); + for (int r = 0; r < nhost; ++r) { + for (int i = 0; i < have_r.cnt[r]; ++i) { + T key = have_r.data[have_r.displ[r] + i]; + if (havekey2rank.find(key) != havekey2rank.end()) { + hoc_execerr_ext( + "internal error in have_to_want: key %lld owned by multiple ranks\n", + (long long) key); + } + havekey2rank[key] = r; + } + } + } + + auto [want_s, want_r] = rendezvous_rank_get(want, alltoall_function); + + // 2) Rendezvous rank matches have and want. + // we already have made the havekey2rank map. + // Create an array parallel to want_r_data which contains the ranks that + // have that data. + int n = want_r.displ[nhost]; + std::vector want_r_ownerranks(n); + for (int r = 0; r < nhost; ++r) { + for (int i = 0; i < want_r.cnt[r]; ++i) { + int ix = want_r.displ[r] + i; + T key = want_r.data[ix]; + auto search = havekey2rank.find(key); + if (search == havekey2rank.end()) { + hoc_execerr_ext( + "internal error in have_to_want: key = %lld is wanted but does not exist\n", + (long long) key); + } + want_r_ownerranks[ix] = search->second; + } + } + + // 3) Rendezvous ranks tell the want ranks which ranks own the keys + // The ranks that want keys need to know the ranks that own those keys. + // The want_s_ownerranks will be parallel to the want_s_data. + // That is, each item defines the rank from which information associated + // with that key is coming from + std::vector want_s_ownerranks(want_s.displ[nhost]); + if (nrn_sparse_partrans > 0) { + nrnmpi_int_alltoallv_sparse(want_r_ownerranks.data(), + want_r.cnt.data(), + want_r.displ.data(), + want_s_ownerranks.data(), + want_s.cnt.data(), + want_s.displ.data()); + } else { + nrnmpi_int_alltoallv(want_r_ownerranks.data(), + want_r.cnt.data(), + want_r.displ.data(), + want_s_ownerranks.data(), + want_s.cnt.data(), + want_s.displ.data()); + } + + // 4) Ranks that want tell owner ranks where to send. + // Finished with the rendezvous ranks. The ranks that want keys know the + // owner ranks for those keys. The next step is for the want ranks to + // tell the owner ranks where to send. + // The parallel want_s_ownerranks and want_s_data are now uselessly ordered + // by rendezvous rank. Reorganize so that want ranks can tell owner ranks + // what they want. + n = want_s.displ[nhost]; + for (int i = 0; i < nhost; ++i) { + want_s.cnt[i] = 0; + } + std::vector old_want_s_data(n); + std::swap(old_want_s_data, want_s.data); + // compute the counts + for (int i = 0; i < n; ++i) { + int r = want_s_ownerranks[i]; + ++want_s.cnt[r]; + } + want_s.displ = cnt2displ(want_s.cnt); + for (int i = 0; i < nhost; ++i) { + want_s.cnt[i] = 0; + } // recount while filling + for (int i = 0; i < n; ++i) { + int r = want_s_ownerranks[i]; + T key = old_want_s_data[i]; + want_s.data[want_s.displ[r] + want_s.cnt[r]] = key; + ++want_s.cnt[r]; + } + + Data new_want_r{}; + new_want_r.cnt = srccnt2destcnt(want_s.cnt); + new_want_r.displ = cnt2displ(new_want_r.cnt); + new_want_r.data.resize(new_want_r.displ[nhost]); + alltoall_function(want_s, new_want_r); + // now the want_r_data on the have_ranks are grouped according to the ranks + // that want those keys. + + return {new_want_r, want_s}; +} diff --git a/src/nrniv/partrans.cpp b/src/nrniv/partrans.cpp index 1f34f9bbf2..1ebfdf335d 100644 --- a/src/nrniv/partrans.cpp +++ b/src/nrniv/partrans.cpp @@ -17,23 +17,48 @@ #include #include +#if NRNMPI +#include "have2want.hpp" +#endif + + #if NRNLONGSGID #if NRNMPI -static void sgid_alltoallv(sgid_t* s, int* scnt, int* sdispl, sgid_t* r, int* rcnt, int* rdispl) { +static void sgid_alltoallv(Data& s, Data& r) { if (nrn_sparse_partrans > 0) { - nrnmpi_long_alltoallv_sparse(s, scnt, sdispl, r, rcnt, rdispl); + nrnmpi_long_alltoallv_sparse(s.data.data(), + s.cnt.data(), + s.displ.data(), + r.data.data(), + r.cnt.data(), + r.displ.data()); } else { - nrnmpi_long_alltoallv(s, scnt, sdispl, r, rcnt, rdispl); + nrnmpi_long_alltoallv(s.data.data(), + s.cnt.data(), + s.displ.data(), + r.data.data(), + r.cnt.data(), + r.displ.data()); } } #endif // NRNMPI #else // not NRNLONGSGID #if NRNMPI -static void sgid_alltoallv(sgid_t* s, int* scnt, int* sdispl, sgid_t* r, int* rcnt, int* rdispl) { +static void sgid_alltoallv(Data& s, Data& r) { if (nrn_sparse_partrans > 0) { - nrnmpi_int_alltoallv_sparse(s, scnt, sdispl, r, rcnt, rdispl); + nrnmpi_int_alltoallv_sparse(s.data.data(), + s.cnt.data(), + s.displ.data(), + r.data.data(), + r.cnt.data(), + r.displ.data()); } else { - nrnmpi_int_alltoallv(s, scnt, sdispl, r, rcnt, rdispl); + nrnmpi_int_alltoallv(s.data.data(), + s.cnt.data(), + s.displ.data(), + r.data.data(), + r.cnt.data(), + r.displ.data()); } } #endif // NRNMPI @@ -152,8 +177,12 @@ static std::vector> poutsrc_; // prior t // to proper place in // outsrc_buf_ static int* poutsrc_indices_; // for recalc pointers -static int insrc_buf_size_, *insrccnt_, *insrcdspl_; -static int outsrc_buf_size_, *outsrccnt_, *outsrcdspl_; +static int insrc_buf_size_; +static std::vector insrccnt_; +static std::vector insrcdspl_; +static int outsrc_buf_size_; +static std::vector outsrccnt_; +static std::vector outsrcdspl_; static MapSgid2Int sid2insrc_; // received interprocessor sid data is // associated with which insrc_buf index. Created by nrnmpi_setup_transfer // and used by mk_ttd @@ -522,11 +551,19 @@ static void mpi_transfer() { if (nrnmpi_numprocs > 1) { double wt = nrnmpi_wtime(); if (nrn_sparse_partrans > 0) { - nrnmpi_dbl_alltoallv_sparse( - outsrc_buf_, outsrccnt_, outsrcdspl_, insrc_buf_, insrccnt_, insrcdspl_); + nrnmpi_dbl_alltoallv_sparse(outsrc_buf_, + outsrccnt_.data(), + outsrcdspl_.data(), + insrc_buf_, + insrccnt_.data(), + insrcdspl_.data()); } else { - nrnmpi_dbl_alltoallv( - outsrc_buf_, outsrccnt_, outsrcdspl_, insrc_buf_, insrccnt_, insrcdspl_); + nrnmpi_dbl_alltoallv(outsrc_buf_, + outsrccnt_.data(), + outsrcdspl_.data(), + insrc_buf_, + insrccnt_.data(), + insrcdspl_.data()); } nrnmpi_transfer_wait_ += nrnmpi_wtime() - wt; errno = 0; @@ -577,15 +614,6 @@ static void thread_transfer(NrnThread* _nt) { // " But this was a mistake as many mpi implementations do not allow overlap // of send and receive buffers. -// 22-08-2014 For setup of the All2allv pattern, use the rendezvous rank -// idiom. -#define HAVEWANT_t sgid_t -#define HAVEWANT_alltoallv sgid_alltoallv -#define HAVEWANT2Int MapSgid2Int -#if NRNMPI -#include "have2want.cpp" -#endif - void nrnmpi_setup_transfer() { #if !NRNMPI if (nrnmpi_numprocs > 1) { @@ -611,10 +639,14 @@ void nrnmpi_setup_transfer() { return; } if (nrnmpi_numprocs > 1) { - delete[] std::exchange(insrccnt_, nullptr); - delete[] std::exchange(insrcdspl_, nullptr); - delete[] std::exchange(outsrccnt_, nullptr); - delete[] std::exchange(outsrcdspl_, nullptr); + insrccnt_.clear(); + insrccnt_.shrink_to_fit(); + insrcdspl_.clear(); + insrcdspl_.shrink_to_fit(); + outsrccnt_.clear(); + outsrccnt_.shrink_to_fit(); + outsrcdspl_.clear(); + outsrcdspl_.shrink_to_fit(); // This is an old comment prior to using the want_to_have rendezvous // rank function in want2have.cpp. The old method did not scale // to more sgids than could fit on a single rank, because @@ -649,78 +681,54 @@ void nrnmpi_setup_transfer() { // sids needed by this machine. The 'seen' table values are unused // but the keys are all the (unique) sgid needed by this process. // At the end seen is in fact what we want for sid2insrc_. - int needsrc_cnt = 0; int szalloc = targets_.size(); szalloc = szalloc ? szalloc : 1; // At the moment sid2insrc_ is serving as 'seen' sid2insrc_.clear(); - sid2insrc_.reserve(szalloc); // for single counting - sgid_t* needsrc = new sgid_t[szalloc]; // more than we need + sid2insrc_.reserve(szalloc); // for single counting + std::vector needsrc{}; for (size_t i = 0; i < sgid2targets_.size(); ++i) { sgid_t sid = sgid2targets_[i]; auto search = sid2insrc_.find(sid); if (search == sid2insrc_.end()) { sid2insrc_[sid] = 0; // at the moment, value does not matter - needsrc[needsrc_cnt++] = sid; + needsrc.push_back(sid); } } // 1 continued) Create an array of sources this rank owns. // This already exists as a vector in the SgidList sgids_ but // that is private so go ahead and copy. - sgid_t* ownsrc = new sgid_t[sgids_.size() + 1]; // not 0 length if count is 0 - for (size_t i = 0; i < sgids_.size(); ++i) { - ownsrc[i] = sgids_[i]; - } + std::vector ownsrc = sgids_; // 2) Call the have_to_want function. - sgid_t* send_to_want; - int *send_to_want_cnt, *send_to_want_displ; - sgid_t* recv_from_have; - int *recv_from_have_cnt, *recv_from_have_displ; - - have_to_want(ownsrc, - sgids_.size(), - needsrc, - needsrc_cnt, - send_to_want, - send_to_want_cnt, - send_to_want_displ, - recv_from_have, - recv_from_have_cnt, - recv_from_have_displ, - default_rendezvous); + auto [send_to_want, recv_from_have] = have_to_want(ownsrc, needsrc, sgid_alltoallv); // sanity check. all the sgids we are asked to send, we actually have - int n = send_to_want_displ[nhost]; + int n = send_to_want.displ[nhost]; // sanity check. all the sgids we receive, we actually need. // also set the sid2insrc_ value to the proper recv_from_have index. - n = recv_from_have_displ[nhost]; + n = recv_from_have.displ[nhost]; for (int i = 0; i < n; ++i) { - sgid_t sgid = recv_from_have[i]; + sgid_t sgid = recv_from_have.data[i]; nrn_assert(sid2insrc_.find(sgid) != sid2insrc_.end()); sid2insrc_[sgid] = i; } - // clean up a little - delete[] std::exchange(ownsrc, nullptr); - delete[] std::exchange(needsrc, nullptr); - delete[] std::exchange(recv_from_have, nullptr); - // 3) First return triple creates the proper outsrc_buf_. // Now that we know what machines are interested in our sids... // construct outsrc_buf, outsrc_buf_size, outsrccnt_, outsrcdspl_ // and poutsrc_; - outsrccnt_ = send_to_want_cnt; - outsrcdspl_ = send_to_want_displ; + std::swap(outsrccnt_, send_to_want.cnt); + std::swap(outsrcdspl_, send_to_want.displ); outsrc_buf_size_ = outsrcdspl_[nrnmpi_numprocs]; szalloc = std::max(1, outsrc_buf_size_); outsrc_buf_ = new double[szalloc]; poutsrc_.resize(szalloc); poutsrc_indices_ = new int[szalloc]; for (int i = 0; i < outsrc_buf_size_; ++i) { - sgid_t sid = send_to_want[i]; + sgid_t sid = send_to_want.data[i]; auto search = sgid2srcindex_.find(sid); nrn_assert(search != sgid2srcindex_.end()); Node* nd = visources_[search->second]; @@ -735,12 +743,11 @@ void nrnmpi_setup_transfer() { poutsrc_indices_[i] = search->second; outsrc_buf_[i] = double(sid); // see step 5 } - delete[] send_to_want; // 4) The second triple is creates the insrc_buf_. // From the recv_from_have and sid2insrc_ table, construct the insrc... - insrccnt_ = recv_from_have_cnt; - insrcdspl_ = recv_from_have_displ; + std::swap(insrccnt_, recv_from_have.cnt); + std::swap(insrcdspl_, recv_from_have.displ); insrc_buf_size_ = insrcdspl_[nrnmpi_numprocs]; szalloc = insrc_buf_size_ ? insrc_buf_size_ : 1; insrc_buf_ = new double[szalloc];