From 35f3b3fcee054813b3375b3cf41191c678003508 Mon Sep 17 00:00:00 2001 From: Tao He Date: Sun, 15 Jan 2023 17:58:53 +0800 Subject: [PATCH] Less copy between in-process memory and vineyard during building a fragment (#1154) Signed-off-by: Tao He --- modules/basic/ds/arrow.cc | 53 ++++++ modules/basic/ds/arrow.h | 43 ++++- modules/basic/ds/arrow_utils.h | 8 + modules/graph/fragment/arrow_fragment.h | 2 +- .../fragment/arrow_fragment.vineyard-mod | 2 +- .../fragment/arrow_fragment_builder_impl.h | 122 +++++-------- modules/graph/fragment/arrow_fragment_impl.h | 36 ++-- modules/graph/fragment/property_graph_utils.h | 15 +- .../fragment/property_graph_utils_impl.h | 162 +++++++----------- .../fragment/property_graph_utils_uint32.cc | 15 +- .../fragment/property_graph_utils_uint64.cc | 15 +- modules/graph/loader/arrow_fragment_loader.cc | 16 +- .../graph/loader/arrow_fragment_loader_impl.h | 4 +- src/client/client.cc | 3 +- src/client/ds/object_meta.cc | 35 +++- src/client/ds/object_meta.h | 56 +++++- src/common/util/env.cc | 6 + 17 files changed, 349 insertions(+), 244 deletions(-) diff --git a/modules/basic/ds/arrow.cc b/modules/basic/ds/arrow.cc index cc93be3b..cd638f00 100644 --- a/modules/basic/ds/arrow.cc +++ b/modules/basic/ds/arrow.cc @@ -314,6 +314,59 @@ template class NumericArrayBuilder; template class NumericArrayBuilder; template class NumericArrayBuilder; +template +FixedNumericArrayBuilder::FixedNumericArrayBuilder(Client& client, + const size_t size) + : NumericArrayBaseBuilder(client), size_(size) { + if (size_ > 0) { + VINEYARD_CHECK_OK(client.CreateBlob(size_ * sizeof(T), writer_)); + data_ = reinterpret_cast(writer_->data()); + } +} + +template +size_t FixedNumericArrayBuilder::size() const { + return size_; +} + +template +T* FixedNumericArrayBuilder::MutablePointer(int64_t i) const { + if (data_) { + return data_ + i; + } + return nullptr; +} + +template +T* FixedNumericArrayBuilder::data() const { + return data_; +} + +template +Status FixedNumericArrayBuilder::Build(Client& client) { + this->set_length_(size_); + this->set_null_count_(0); + this->set_offset_(0); + if (size_ > 0) { + this->set_buffer_(std::move(writer_)); + } else { + this->set_buffer_(Blob::MakeEmpty(client)); + } + this->set_null_bitmap_(Blob::MakeEmpty(client)); + return Status::OK(); +} + +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; +template class FixedNumericArrayBuilder; + BooleanArrayBuilder::BooleanArrayBuilder(Client& client) : BooleanArrayBaseBuilder(client) { std::shared_ptr array; diff --git a/modules/basic/ds/arrow.h b/modules/basic/ds/arrow.h index fe93ab8c..61b2ba09 100644 --- a/modules/basic/ds/arrow.h +++ b/modules/basic/ds/arrow.h @@ -77,6 +77,34 @@ class NumericArrayBuilder : public NumericArrayBaseBuilder { std::vector> arrays_; }; +/** + * @brief FixedNumericArrayBuilder is designed for building Arrow numeric + * arrays with known size. It is useful for allocating buffers directly + * on the vineyard's shared memory. + * + * @tparam T + */ +template +class FixedNumericArrayBuilder : public NumericArrayBaseBuilder { + public: + using ArrayType = ArrowArrayType; + + FixedNumericArrayBuilder(Client& client, const size_t size); + + size_t size() const; + + T* MutablePointer(int64_t i) const; + + T* data() const; + + Status Build(Client& client) override; + + private: + size_t size_ = 0; + std::unique_ptr writer_ = nullptr; + T* data_ = nullptr; +}; + using Int8Builder = NumericArrayBuilder; using Int16Builder = NumericArrayBuilder; using Int32Builder = NumericArrayBuilder; @@ -88,6 +116,17 @@ using UInt64Builder = NumericArrayBuilder; using FloatBuilder = NumericArrayBuilder; using DoubleBuilder = NumericArrayBuilder; +using FixedInt8Builder = FixedNumericArrayBuilder; +using FixedInt16Builder = FixedNumericArrayBuilder; +using FixedInt32Builder = FixedNumericArrayBuilder; +using FixedInt64Builder = FixedNumericArrayBuilder; +using FixedUInt8Builder = FixedNumericArrayBuilder; +using FixedUInt16Builder = FixedNumericArrayBuilder; +using FixedUInt32Builder = FixedNumericArrayBuilder; +using FixedUInt64Builder = FixedNumericArrayBuilder; +using FixedFloatBuilder = FixedNumericArrayBuilder; +using FixedDoubleBuilder = FixedNumericArrayBuilder; + /** * @brief BooleanArrayBuilder is designed for constructing Arrow arrays of * boolean data type @@ -194,13 +233,15 @@ class PodArrayBuilder : public FixedSizeBinaryArrayBaseBuilder { } } - T* MutablePointer(int64_t i) { + T* MutablePointer(int64_t i) const { if (data_) { return data_ + i; } return nullptr; } + T* data() const { return data_; } + size_t size() const { return size_; } Status Build(Client& client) override { diff --git a/modules/basic/ds/arrow_utils.h b/modules/basic/ds/arrow_utils.h index 3dade364..619f76a5 100644 --- a/modules/basic/ds/arrow_utils.h +++ b/modules/basic/ds/arrow_utils.h @@ -50,6 +50,9 @@ class NumericArray; template class NumericArrayBuilder; +template +class FixedNumericArrayBuilder; + #define CONVERT_TO_ARROW_TYPE(type, data_type, array_type, builder_type, \ type_value) \ template <> \ @@ -59,6 +62,7 @@ class NumericArrayBuilder; using VineyardArrayType = NumericArray; \ using BuilderType = builder_type; \ using VineyardBuilderType = NumericArrayBuilder; \ + using FixedVineyardBuilderType = FixedNumericArrayBuilder; \ static std::shared_ptr TypeValue() { return type_value; } \ }; @@ -79,6 +83,10 @@ template using ArrowVineyardBuilderType = typename ConvertToArrowType::VineyardBuilderType; +template +using FixedArrowVineyardBuilderType = + typename ConvertToArrowType::FixedVineyardBuilderType; + CONVERT_TO_ARROW_TYPE(void, arrow::NullType, arrow::NullArray, arrow::NullBuilder, arrow::null()) CONVERT_TO_ARROW_TYPE(bool, arrow::BooleanType, arrow::BooleanArray, diff --git a/modules/graph/fragment/arrow_fragment.h b/modules/graph/fragment/arrow_fragment.h index cd646e48..b6de54a2 100644 --- a/modules/graph/fragment/arrow_fragment.h +++ b/modules/graph/fragment/arrow_fragment.h @@ -112,7 +112,7 @@ class BasicArrowFragmentBuilder std::vector>>> ie_lists_, oe_lists_; - std::vector>> + std::vector>> ie_offsets_lists_, oe_offsets_lists_; std::shared_ptr vm_ptr_; diff --git a/modules/graph/fragment/arrow_fragment.vineyard-mod b/modules/graph/fragment/arrow_fragment.vineyard-mod index 62f3ce2c..560aa4b0 100644 --- a/modules/graph/fragment/arrow_fragment.vineyard-mod +++ b/modules/graph/fragment/arrow_fragment.vineyard-mod @@ -678,7 +678,7 @@ class [[vineyard]] ArrowFragment vineyard::Client & client, std::vector>>> & oe_lists, - std::vector>> & + std::vector>> & oe_offsets_lists, int concurrency, bool& is_multigraph); diff --git a/modules/graph/fragment/arrow_fragment_builder_impl.h b/modules/graph/fragment/arrow_fragment_builder_impl.h index b8e2bc9b..f0af3ddd 100644 --- a/modules/graph/fragment/arrow_fragment_builder_impl.h +++ b/modules/graph/fragment/arrow_fragment_builder_impl.h @@ -273,9 +273,9 @@ ArrowFragment::AddNewVertexEdgeLabels( ie_lists(total_vertex_label_num); std::vector>>> oe_lists(total_vertex_label_num); - std::vector>> ie_offsets_lists( + std::vector>> ie_offsets_lists( total_vertex_label_num); - std::vector>> oe_offsets_lists( + std::vector>> oe_offsets_lists( total_vertex_label_num); for (label_id_t v_label = 0; v_label < total_vertex_label_num; ++v_label) { @@ -292,7 +292,9 @@ ArrowFragment::AddNewVertexEdgeLabels( vid_t prev_offset_size = tvnums_[v_label] + 1; vid_t cur_offset_size = tvnums[v_label] + 1; if (directed_) { - std::vector offsets(cur_offset_size); + ie_offsets_lists[v_label][e_label] = + std::make_shared(client, cur_offset_size); + int64_t* offsets = ie_offsets_lists[v_label][e_label]->data(); const int64_t* offset_array = ie_offsets_ptr_lists_[v_label][e_label]; for (vid_t k = 0; k < prev_offset_size; ++k) { offsets[k] = offset_array[k]; @@ -300,11 +302,10 @@ ArrowFragment::AddNewVertexEdgeLabels( for (vid_t k = prev_offset_size; k < cur_offset_size; ++k) { offsets[k] = offsets[k - 1]; } - arrow::Int64Builder builder; - ARROW_OK_OR_RAISE(builder.AppendValues(offsets)); - ARROW_OK_OR_RAISE(builder.Finish(&ie_offsets_lists[v_label][e_label])); } - std::vector offsets(cur_offset_size); + oe_offsets_lists[v_label][e_label] = + std::make_shared(client, cur_offset_size); + int64_t* offsets = oe_offsets_lists[v_label][e_label]->data(); const int64_t* offset_array = oe_offsets_ptr_lists_[v_label][e_label]; for (size_t k = 0; k < prev_offset_size; ++k) { offsets[k] = offset_array[k]; @@ -312,9 +313,6 @@ ArrowFragment::AddNewVertexEdgeLabels( for (size_t k = prev_offset_size; k < cur_offset_size; ++k) { offsets[k] = offsets[k - 1]; } - arrow::Int64Builder builder; - ARROW_OK_OR_RAISE(builder.AppendValues(offsets)); - ARROW_OK_OR_RAISE(builder.Finish(&oe_offsets_lists[v_label][e_label])); } } @@ -323,9 +321,9 @@ ArrowFragment::AddNewVertexEdgeLabels( total_vertex_label_num); std::vector>> sub_oe_lists( total_vertex_label_num); - std::vector> sub_ie_offset_lists( + std::vector> sub_ie_offset_lists( total_vertex_label_num); - std::vector> sub_oe_offset_lists( + std::vector> sub_oe_offset_lists( total_vertex_label_num); // Process v_num...total_v_num X 0...e_num part. @@ -335,25 +333,20 @@ ArrowFragment::AddNewVertexEdgeLabels( v_label < total_vertex_label_num; ++v_label) { sub_ie_lists[v_label] = std::make_shared>(client, 0); - arrow::Int64Builder int64_builder; - std::vector offset_vec(tvnums[v_label] + 1, 0); - ARROW_OK_OR_RAISE(int64_builder.AppendValues(offset_vec)); - std::shared_ptr ie_offset_array; - ARROW_OK_OR_RAISE(int64_builder.Finish(&ie_offset_array)); - sub_ie_offset_lists[v_label] = ie_offset_array; + sub_ie_offset_lists[v_label] = + std::make_shared(client, tvnums[v_label] + 1); + memset(sub_ie_offset_lists[v_label]->data(), 0x00, + sizeof(int64_t) * (tvnums[v_label] + 1)); } } for (label_id_t v_label = vertex_label_num_; v_label < total_vertex_label_num; ++v_label) { sub_oe_lists[v_label] = std::make_shared>(client, 0); - - arrow::Int64Builder int64_builder; - std::vector offset_vec(tvnums[v_label] + 1, 0); - ARROW_OK_OR_RAISE(int64_builder.AppendValues(offset_vec)); - std::shared_ptr oe_offset_array; - ARROW_OK_OR_RAISE(int64_builder.Finish(&oe_offset_array)); - sub_oe_offset_lists[v_label] = oe_offset_array; + sub_oe_offset_lists[v_label] = + std::make_shared(client, tvnums[v_label] + 1); + memset(sub_oe_offset_lists[v_label]->data(), 0x00, + sizeof(int64_t) * (tvnums[v_label] + 1)); } } else { auto cur_label_index = e_label - edge_label_num_; @@ -517,16 +510,12 @@ ArrowFragment::AddNewVertexEdgeLabels( if (!(i < vertex_label_num_ && j < edge_label_num_)) { builder.set_ie_lists_(i, j, ie_lists[i][j]); } - vineyard::NumericArrayBuilder ieo_builder( - *client, ie_offsets_lists[i][j]); - builder.set_ie_offsets_lists_(i, j, ieo_builder.Seal(*client)); + builder.set_ie_offsets_lists_(i, j, ie_offsets_lists[i][j]); } if (!(i < vertex_label_num_ && j < edge_label_num_)) { builder.set_oe_lists_(i, j, oe_lists[i][j]); } - vineyard::NumericArrayBuilder oeo_builder( - *client, oe_offsets_lists[i][j]); - builder.set_oe_offsets_lists_(i, j, oeo_builder.Seal(*client)); + builder.set_oe_offsets_lists_(i, j, oe_offsets_lists[i][j]); return Status::OK(); }; tg.AddTask(fn, &client); @@ -754,9 +743,9 @@ ArrowFragment::AddNewEdgeLabels( extra_ovgid_lists[i].reset(); // release the reference } - std::vector>> + std::vector>> ie_offsets_lists_expanded(vertex_label_num_); - std::vector>> + std::vector>> oe_offsets_lists_expanded(vertex_label_num_); for (label_id_t v_label = 0; v_label < vertex_label_num_; ++v_label) { @@ -770,7 +759,9 @@ ArrowFragment::AddNewEdgeLabels( vid_t prev_offset_size = tvnums_[v_label] + 1; vid_t current_offset_size = tvnums[v_label] + 1; if (directed_) { - std::vector offsets(current_offset_size); + ie_offsets_lists_expanded[v_label][e_label] = + std::make_shared(client, current_offset_size); + int64_t* offsets = ie_offsets_lists_expanded[v_label][e_label]->data(); const int64_t* offset_array = ie_offsets_ptr_lists_[v_label][e_label]; for (vid_t k = 0; k < prev_offset_size; ++k) { offsets[k] = offset_array[k]; @@ -778,12 +769,10 @@ ArrowFragment::AddNewEdgeLabels( for (vid_t k = prev_offset_size; k < current_offset_size; ++k) { offsets[k] = offsets[k - 1]; } - arrow::Int64Builder builder; - ARROW_OK_OR_RAISE(builder.AppendValues(offsets)); - ARROW_OK_OR_RAISE( - builder.Finish(&ie_offsets_lists_expanded[v_label][e_label])); } - std::vector offsets(current_offset_size); + oe_offsets_lists_expanded[v_label][e_label] = + std::make_shared(client, current_offset_size); + int64_t* offsets = oe_offsets_lists_expanded[v_label][e_label]->data(); const int64_t* offset_array = oe_offsets_ptr_lists_[v_label][e_label]; for (size_t k = 0; k < prev_offset_size; ++k) { offsets[k] = offset_array[k]; @@ -791,10 +780,6 @@ ArrowFragment::AddNewEdgeLabels( for (size_t k = prev_offset_size; k < current_offset_size; ++k) { offsets[k] = offsets[k - 1]; } - arrow::Int64Builder builder; - ARROW_OK_OR_RAISE(builder.AppendValues(offsets)); - ARROW_OK_OR_RAISE( - builder.Finish(&oe_offsets_lists_expanded[v_label][e_label])); } } // Gather all local id of new edges. @@ -823,9 +808,9 @@ ArrowFragment::AddNewEdgeLabels( ie_lists(vertex_label_num_); std::vector>>> oe_lists(vertex_label_num_); - std::vector>> ie_offsets_lists( + std::vector>> ie_offsets_lists( vertex_label_num_); - std::vector>> oe_offsets_lists( + std::vector>> oe_offsets_lists( vertex_label_num_); for (label_id_t v_label = 0; v_label < vertex_label_num_; ++v_label) { @@ -842,9 +827,9 @@ ArrowFragment::AddNewEdgeLabels( vertex_label_num_); std::vector>> sub_oe_lists( vertex_label_num_); - std::vector> sub_ie_offset_lists( + std::vector> sub_ie_offset_lists( vertex_label_num_); - std::vector> sub_oe_offset_lists( + std::vector> sub_oe_offset_lists( vertex_label_num_); if (directed_) { generate_directed_csr( @@ -965,18 +950,11 @@ ArrowFragment::AddNewEdgeLabels( label_id_t edge_label_id = edge_label_num_ + j; if (directed_) { builder.set_ie_lists_(i, edge_label_id, ie_lists[i][j]); - - vineyard::NumericArrayBuilder ieo_builder( - *client, ie_offsets_lists[i][j]); builder.set_ie_offsets_lists_(i, edge_label_id, - ieo_builder.Seal(*client)); + ie_offsets_lists[i][j]); } builder.set_oe_lists_(i, edge_label_id, oe_lists[i][j]); - - vineyard::NumericArrayBuilder oeo_builder( - *client, oe_offsets_lists[i][j]); - builder.set_oe_offsets_lists_(i, edge_label_id, - oeo_builder.Seal(*client)); + builder.set_oe_offsets_lists_(i, edge_label_id, oe_offsets_lists[i][j]); return Status::OK(); }; tg.AddTask(fn, &client); @@ -984,22 +962,16 @@ ArrowFragment::AddNewEdgeLabels( } for (label_id_t i = 0; i < vertex_label_num_; ++i) { for (label_id_t j = 0; j < edge_label_num_; ++j) { - auto fn = [this, &builder, i, j, &ie_offsets_lists_expanded, - &oe_offsets_lists_expanded](Client* client) -> Status { - label_id_t edge_label_id = j; + auto fn = [this, &builder, &ie_offsets_lists_expanded, + &oe_offsets_lists_expanded](Client* client, const label_id_t i, + const label_id_t j) -> Status { if (directed_) { - vineyard::NumericArrayBuilder ieo_builder_expanded( - *client, ie_offsets_lists_expanded[i][j]); - builder.set_ie_offsets_lists_(i, edge_label_id, - ieo_builder_expanded.Seal(*client)); + builder.set_ie_offsets_lists_(i, j, ie_offsets_lists_expanded[i][j]); } - vineyard::NumericArrayBuilder oeo_builder_expanded( - *client, oe_offsets_lists_expanded[i][j]); - builder.set_oe_offsets_lists_(i, edge_label_id, - oeo_builder_expanded.Seal(*client)); + builder.set_oe_offsets_lists_(i, j, oe_offsets_lists_expanded[i][j]); return Status::OK(); }; - tg.AddTask(fn, &client); + tg.AddTask(fn, &client, i, j); } } tg.TakeResults(); @@ -1087,15 +1059,13 @@ vineyard::Status BasicArrowFragmentBuilder::Build( auto fn = [this, i, j](Client* client) -> Status { if (this->directed_) { this->set_ie_lists_(i, j, ie_lists_[i][j]->Seal(*client)); - vineyard::NumericArrayBuilder ieo( - *client, std::move(ie_offsets_lists_[i][j])); - this->set_ie_offsets_lists_(i, j, ieo.Seal(*client)); + this->set_ie_offsets_lists_(i, j, + ie_offsets_lists_[i][j]->Seal(*client)); } { this->set_oe_lists_(i, j, oe_lists_[i][j]->Seal(*client)); - vineyard::NumericArrayBuilder oeo( - *client, std::move(oe_offsets_lists_[i][j])); - this->set_oe_offsets_lists_(i, j, oeo.Seal(*client)); + this->set_oe_offsets_lists_(i, j, + oe_offsets_lists_[i][j]->Seal(*client)); } return Status::OK(); }; @@ -1237,9 +1207,9 @@ BasicArrowFragmentBuilder::initEdges( this->vertex_label_num_); std::vector>> sub_oe_lists( this->vertex_label_num_); - std::vector> sub_ie_offset_lists( + std::vector> sub_ie_offset_lists( this->vertex_label_num_); - std::vector> sub_oe_offset_lists( + std::vector> sub_oe_offset_lists( this->vertex_label_num_); if (this->directed_) { generate_directed_csr( diff --git a/modules/graph/fragment/arrow_fragment_impl.h b/modules/graph/fragment/arrow_fragment_impl.h index 00cf2a18..66701119 100644 --- a/modules/graph/fragment/arrow_fragment_impl.h +++ b/modules/graph/fragment/arrow_fragment_impl.h @@ -326,7 +326,7 @@ ArrowFragment::TransformDirection( std::vector>>> oe_lists(vertex_label_num_); - std::vector>> oe_offsets_lists( + std::vector>> oe_offsets_lists( vertex_label_num_); for (label_id_t v_label = 0; v_label < vertex_label_num_; ++v_label) { @@ -342,9 +342,7 @@ ArrowFragment::TransformDirection( for (label_id_t i = 0; i < vertex_label_num_; ++i) { for (label_id_t j = 0; j < edge_label_num_; ++j) { builder.set_oe_lists_(i, j, oe_lists[i][j]); - vineyard::NumericArrayBuilder oeo_builder( - client, oe_offsets_lists[i][j]); - builder.set_oe_offsets_lists_(i, j, oeo_builder.Seal(client)); + builder.set_oe_offsets_lists_(i, j, oe_offsets_lists[i][j]); } } builder.set_is_multigraph_(is_multigraph); @@ -608,7 +606,7 @@ void ArrowFragment::directedCSR2Undirected( vineyard::Client& client, std::vector>>>& oe_lists, - std::vector>>& + std::vector>>& oe_offsets_lists, int concurrency, bool& is_multigraph) { for (label_id_t v_label = 0; v_label < vertex_label_num_; ++v_label) { @@ -623,33 +621,33 @@ void ArrowFragment::directedCSR2Undirected( std::make_shared>( client, ie_offset[tvnums_[v_label]] + oe_offset[tvnums_[v_label]]); + auto offsets_builder = + std::make_shared(client, tvnums_[v_label] + 1); + nbr_unit_t* data = edge_builder->MutablePointer(0); - arrow::Int64Builder offset_builder; - CHECK_ARROW_ERROR(offset_builder.Append(0)); + int64_t* offsets = offsets_builder->MutablePointer(0); + offsets[0] = 0; + size_t edge_offset = 0; - for (size_t offset = 0; offset < static_cast(tvnums_[v_label]); - ++offset) { - for (size_t k = ie_offset[offset]; - k < static_cast(ie_offset[offset + 1]); ++k) { + for (size_t v = 0; v < static_cast(tvnums_[v_label]); ++v) { + for (size_t k = ie_offset[v]; k < static_cast(ie_offset[v + 1]); + ++k) { data[edge_offset++] = ie[k]; } - for (int k = oe_offset[offset]; k < oe_offset[offset + 1]; ++k) { + for (int k = oe_offset[v]; k < oe_offset[v + 1]; ++k) { data[edge_offset++] = oe[k]; } - CHECK_ARROW_ERROR(offset_builder.Append(edge_offset)); + offsets[v + 1 /* offsets array */] = edge_offset; } - CHECK_ARROW_ERROR( - offset_builder.Finish(&oe_offsets_lists[v_label][e_label])); - - sort_edges_with_respect_to_vertex(*edge_builder, - oe_offsets_lists[v_label][e_label], + sort_edges_with_respect_to_vertex(*edge_builder, offsets_builder->data(), tvnums_[v_label], concurrency); if (!is_multigraph) { - check_is_multigraph(*edge_builder, oe_offsets_lists[v_label][e_label], + check_is_multigraph(*edge_builder, offsets_builder->data(), tvnums_[v_label], concurrency, is_multigraph); } oe_lists[v_label][e_label] = edge_builder; + oe_offsets_lists[v_label][e_label] = offsets_builder; } } } diff --git a/modules/graph/fragment/property_graph_utils.h b/modules/graph/fragment/property_graph_utils.h index a7d3774b..34fe954e 100644 --- a/modules/graph/fragment/property_graph_utils.h +++ b/modules/graph/fragment/property_graph_utils.h @@ -98,14 +98,13 @@ template void sort_edges_with_respect_to_vertex( vineyard::PodArrayBuilder>& builder, - std::shared_ptr offsets, VID_T tvnum, int concurrency); + const int64_t* offsets, VID_T tvnum, int concurrency); template void check_is_multigraph( vineyard::PodArrayBuilder>& builder, - std::shared_ptr offsets, VID_T tvnum, int concurrency, - bool& is_multigraph); + const int64_t* offsets, VID_T tvnum, int concurrency, bool& is_multigraph); /** * @brief Generate CSR from given COO. @@ -118,7 +117,7 @@ boost::leaf::result generate_directed_csr( std::vector tvnums, int vertex_label_num, int concurrency, std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); /** @@ -130,10 +129,10 @@ boost::leaf::result generate_directed_csc( int vertex_label_num, int concurrency, std::vector>>>& oedges, - std::vector>& oedge_offsets, + std::vector>& oedge_offsets, std::vector>>>& iedges, - std::vector>& iedge_offsets, + std::vector>& iedge_offsets, bool& is_multigraph); /** @@ -148,7 +147,7 @@ boost::leaf::result generate_undirected_csr( std::vector tvnums, int vertex_label_num, int concurrency, std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); /** @@ -163,7 +162,7 @@ boost::leaf::result generate_undirected_csr_memopt( std::vector tvnums, int vertex_label_num, int concurrency, std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); } // namespace vineyard diff --git a/modules/graph/fragment/property_graph_utils_impl.h b/modules/graph/fragment/property_graph_utils_impl.h index 6272a3e9..fbe5084e 100644 --- a/modules/graph/fragment/property_graph_utils_impl.h +++ b/modules/graph/fragment/property_graph_utils_impl.h @@ -177,73 +177,43 @@ template void sort_edges_with_respect_to_vertex( vineyard::PodArrayBuilder>& builder, - std::shared_ptr offsets, VID_T tvnum, int concurrency) { + const int64_t* offsets, VID_T tvnum, int concurrency) { using nbr_unit_t = property_graph_utils::NbrUnit; - - const int64_t* offsets_ptr = offsets->raw_values(); - if (concurrency == 1) { - for (VID_T i = 0; i < tvnum; ++i) { - nbr_unit_t* begin = builder.MutablePointer(offsets_ptr[i]); - nbr_unit_t* end = builder.MutablePointer(offsets_ptr[i + 1]); - std::sort(begin, end, [](const nbr_unit_t& lhs, const nbr_unit_t& rhs) { - return lhs.vid < rhs.vid; - }); - } - } else { - parallel_for( - static_cast(0), tvnum, - [offsets_ptr, &builder](VID_T i) { - nbr_unit_t* begin = builder.MutablePointer(offsets_ptr[i]); - nbr_unit_t* end = builder.MutablePointer(offsets_ptr[i + 1]); - std::sort(begin, end, - [](const nbr_unit_t& lhs, const nbr_unit_t& rhs) { - return lhs.vid < rhs.vid; - }); - }, - concurrency); - } + parallel_for( + static_cast(0), tvnum, + [offsets, &builder](VID_T i) { + nbr_unit_t* begin = builder.MutablePointer(offsets[i]); + nbr_unit_t* end = builder.MutablePointer(offsets[i + 1]); + std::sort(begin, end, [](const nbr_unit_t& lhs, const nbr_unit_t& rhs) { + return lhs.vid < rhs.vid; + }); + }, + concurrency); } template void check_is_multigraph( vineyard::PodArrayBuilder>& builder, - std::shared_ptr offsets, VID_T tvnum, int concurrency, - bool& is_multigraph) { + const int64_t* offsets, VID_T tvnum, int concurrency, bool& is_multigraph) { using nbr_unit_t = property_graph_utils::NbrUnit; - const int64_t* offsets_ptr = offsets->raw_values(); - if (concurrency == 1) { - for (VID_T i = 0; i < tvnum; ++i) { - nbr_unit_t* begin = builder.MutablePointer(offsets_ptr[i]); - nbr_unit_t* end = builder.MutablePointer(offsets_ptr[i + 1]); - nbr_unit_t* loc = std::adjacent_find( - begin, end, [](const nbr_unit_t& lhs, const nbr_unit_t& rhs) { - return lhs.vid == rhs.vid; - }); - if (loc != end) { - is_multigraph = true; - break; - } - } - } else { - parallel_for( - static_cast(0), tvnum, - [offsets_ptr, &builder, &is_multigraph](VID_T i) { - if (!is_multigraph) { - nbr_unit_t* begin = builder.MutablePointer(offsets_ptr[i]); - nbr_unit_t* end = builder.MutablePointer(offsets_ptr[i + 1]); - nbr_unit_t* loc = std::adjacent_find( - begin, end, [](const nbr_unit_t& lhs, const nbr_unit_t& rhs) { - return lhs.vid == rhs.vid; - }); - if (loc != end) { - __sync_or_and_fetch( - reinterpret_cast(&is_multigraph), 1); - } + parallel_for( + static_cast(0), tvnum, + [offsets, &builder, &is_multigraph](VID_T i) { + if (!is_multigraph) { + nbr_unit_t* begin = builder.MutablePointer(offsets[i]); + nbr_unit_t* end = builder.MutablePointer(offsets[i + 1]); + nbr_unit_t* loc = std::adjacent_find( + begin, end, [](const nbr_unit_t& lhs, const nbr_unit_t& rhs) { + return lhs.vid == rhs.vid; + }); + if (loc != end) { + __sync_or_and_fetch( + reinterpret_cast(&is_multigraph), 1); } - }, - concurrency); - } + } + }, + concurrency); } template @@ -254,7 +224,7 @@ boost::leaf::result generate_directed_csr( std::vector tvnums, int vertex_label_num, int concurrency, std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph) { using nbr_unit_t = property_graph_utils::NbrUnit; @@ -293,14 +263,10 @@ boost::leaf::result generate_directed_csr( concurrency); } // build the arrow's offset array - std::shared_ptr offsets_buffer; - ARROW_OK_ASSIGN_OR_RAISE( - offsets_buffer, arrow::AllocateBuffer((tvnum + 1) * sizeof(int64_t), - arrow::default_memory_pool())); - memcpy(offsets_buffer->mutable_data(), offset_vec.data(), + edge_offsets[v_label] = + std::make_shared(client, tvnum + 1); + memcpy(edge_offsets[v_label]->data(), offset_vec.data(), (tvnum + 1) * sizeof(int64_t)); - edge_offsets[v_label] = std::make_shared( - arrow::int64(), tvnum + 1, offsets_buffer, nullptr, 0, 0); actual_edge_num[v_label] = offset_vec[tvnum]; } for (int v_label = 0; v_label != vertex_label_num; ++v_label) { @@ -341,10 +307,11 @@ boost::leaf::result generate_directed_csr( VLOG(100) << "Finish building the CSR ..." << get_rss_pretty() << ", peak = " << get_peak_rss_pretty(); for (int v_label = 0; v_label != vertex_label_num; ++v_label) { - sort_edges_with_respect_to_vertex(*edges[v_label], edge_offsets[v_label], + sort_edges_with_respect_to_vertex(*edges[v_label], + edge_offsets[v_label]->data(), tvnums[v_label], concurrency); if (!is_multigraph) { - check_is_multigraph(*edges[v_label], edge_offsets[v_label], + check_is_multigraph(*edges[v_label], edge_offsets[v_label]->data(), tvnums[v_label], concurrency, is_multigraph); } } @@ -357,10 +324,10 @@ boost::leaf::result generate_directed_csc( int vertex_label_num, int concurrency, std::vector>>>& oedges, - std::vector>& oedge_offsets, + std::vector>& oedge_offsets, std::vector>>>& iedges, - std::vector>& iedge_offsets, + std::vector>& iedge_offsets, bool& is_multigraph) { using nbr_unit_t = property_graph_utils::NbrUnit; @@ -372,7 +339,7 @@ boost::leaf::result generate_directed_csc( for (int v_label = 0; v_label != vertex_label_num; ++v_label) { const nbr_unit_t* oe = oedges[v_label]->MutablePointer(0); - const int64_t* oe_offsets = oedge_offsets[v_label]->raw_values(); + const int64_t* oe_offsets = oedge_offsets[v_label]->data(); parallel_for( static_cast(0), tvnums[v_label], [°ree, &parser, &oe, &oe_offsets](VID_T src_offset) { @@ -400,14 +367,10 @@ boost::leaf::result generate_directed_csc( concurrency); } // build the arrow's offset array - std::shared_ptr offsets_buffer; - ARROW_OK_ASSIGN_OR_RAISE( - offsets_buffer, arrow::AllocateBuffer((tvnum + 1) * sizeof(int64_t), - arrow::default_memory_pool())); - memcpy(offsets_buffer->mutable_data(), offset_vec.data(), + iedge_offsets[v_label] = + std::make_shared(client, tvnum + 1); + memcpy(iedge_offsets[v_label]->data(), offset_vec.data(), (tvnum + 1) * sizeof(int64_t)); - iedge_offsets[v_label] = std::make_shared( - arrow::int64(), tvnum + 1, offsets_buffer, nullptr, 0, 0); actual_edge_num[v_label] = offset_vec[tvnum]; } for (int v_label = 0; v_label != vertex_label_num; ++v_label) { @@ -420,7 +383,7 @@ boost::leaf::result generate_directed_csc( for (int v_label = 0; v_label != vertex_label_num; ++v_label) { const nbr_unit_t* oe = oedges[v_label]->MutablePointer(0); - const int64_t* oe_offsets = oedge_offsets[v_label]->raw_values(); + const int64_t* oe_offsets = oedge_offsets[v_label]->data(); parallel_for( static_cast(0), tvnums[v_label], [&parser, &v_label, &offsets, &iedges, &oe, @@ -444,10 +407,11 @@ boost::leaf::result generate_directed_csc( VLOG(100) << "Finish building the CSC ..." << get_rss_pretty() << ", peak = " << get_peak_rss_pretty(); for (int v_label = 0; v_label != vertex_label_num; ++v_label) { - sort_edges_with_respect_to_vertex(*iedges[v_label], iedge_offsets[v_label], + sort_edges_with_respect_to_vertex(*iedges[v_label], + iedge_offsets[v_label]->data(), tvnums[v_label], concurrency); if (!is_multigraph) { - check_is_multigraph(*iedges[v_label], iedge_offsets[v_label], + check_is_multigraph(*iedges[v_label], iedge_offsets[v_label]->data(), tvnums[v_label], concurrency, is_multigraph); } } @@ -462,7 +426,7 @@ boost::leaf::result generate_undirected_csr( std::vector tvnums, int vertex_label_num, int concurrency, std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph) { using nbr_unit_t = property_graph_utils::NbrUnit; @@ -508,14 +472,10 @@ boost::leaf::result generate_undirected_csr( concurrency); } // build the arrow's offset array - std::shared_ptr offsets_buffer; - ARROW_OK_ASSIGN_OR_RAISE( - offsets_buffer, arrow::AllocateBuffer((tvnum + 1) * sizeof(int64_t), - arrow::default_memory_pool())); - memcpy(offsets_buffer->mutable_data(), offset_vec.data(), + edge_offsets[v_label] = + std::make_shared(client, tvnum + 1); + memcpy(edge_offsets[v_label]->data(), offset_vec.data(), (tvnum + 1) * sizeof(int64_t)); - edge_offsets[v_label] = std::make_shared( - arrow::int64(), tvnum + 1, offsets_buffer, nullptr, 0, 0); actual_edge_num[v_label] = offset_vec[tvnum]; } @@ -567,10 +527,11 @@ boost::leaf::result generate_undirected_csr( VLOG(100) << "Finish building the CSR ..." << get_rss_pretty() << ", peak = " << get_peak_rss_pretty(); for (int v_label = 0; v_label != vertex_label_num; ++v_label) { - sort_edges_with_respect_to_vertex(*edges[v_label], edge_offsets[v_label], + sort_edges_with_respect_to_vertex(*edges[v_label], + edge_offsets[v_label]->data(), tvnums[v_label], concurrency); if (!is_multigraph) { - check_is_multigraph(*edges[v_label], edge_offsets[v_label], + check_is_multigraph(*edges[v_label], edge_offsets[v_label]->data(), tvnums[v_label], concurrency, is_multigraph); } } @@ -585,7 +546,7 @@ boost::leaf::result generate_undirected_csr_memopt( std::vector tvnums, int vertex_label_num, int concurrency, std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph) { using nbr_unit_t = property_graph_utils::NbrUnit; @@ -631,14 +592,10 @@ boost::leaf::result generate_undirected_csr_memopt( concurrency); } // build the arrow's offset array - std::shared_ptr offsets_buffer; - ARROW_OK_ASSIGN_OR_RAISE( - offsets_buffer, arrow::AllocateBuffer((tvnum + 1) * sizeof(int64_t), - arrow::default_memory_pool())); - memcpy(offsets_buffer->mutable_data(), offset_vec.data(), + edge_offsets[v_label] = + std::make_shared(client, tvnum + 1); + memcpy(edge_offsets[v_label]->data(), offset_vec.data(), (tvnum + 1) * sizeof(int64_t)); - edge_offsets[v_label] = std::make_shared( - arrow::int64(), tvnum + 1, offsets_buffer, nullptr, 0, 0); actual_edge_num[v_label] = offset_vec[tvnum]; } @@ -686,7 +643,7 @@ boost::leaf::result generate_undirected_csr_memopt( for (int v_label = 0; v_label != vertex_label_num; ++v_label) { const nbr_unit_t* oe = edges[v_label]->MutablePointer(0); - const int64_t* oe_offsets = edge_offsets[v_label]->raw_values(); + const int64_t* oe_offsets = edge_offsets[v_label]->data(); parallel_for( static_cast(0), tvnums[v_label], [&parser, &v_label, &csr_offsets, &offsets, &oe_offsets, &edges, @@ -711,10 +668,11 @@ boost::leaf::result generate_undirected_csr_memopt( << ", peak = " << get_peak_rss_pretty(); for (int v_label = 0; v_label != vertex_label_num; ++v_label) { - sort_edges_with_respect_to_vertex(*edges[v_label], edge_offsets[v_label], + sort_edges_with_respect_to_vertex(*edges[v_label], + edge_offsets[v_label]->data(), tvnums[v_label], concurrency); if (!is_multigraph) { - check_is_multigraph(*edges[v_label], edge_offsets[v_label], + check_is_multigraph(*edges[v_label], edge_offsets[v_label]->data(), tvnums[v_label], concurrency, is_multigraph); } } diff --git a/modules/graph/fragment/property_graph_utils_uint32.cc b/modules/graph/fragment/property_graph_utils_uint32.cc index 806b8d9c..7b47e924 100644 --- a/modules/graph/fragment/property_graph_utils_uint32.cc +++ b/modules/graph/fragment/property_graph_utils_uint32.cc @@ -42,13 +42,12 @@ template boost::leaf::result generate_local_id_list( template void sort_edges_with_respect_to_vertex( vineyard::PodArrayBuilder< property_graph_utils::NbrUnit>& builder, - std::shared_ptr offsets, uint32_t tvnum, - int concurrency); + const int64_t* offsets, uint32_t tvnum, int concurrency); template void check_is_multigraph( vineyard::PodArrayBuilder< property_graph_utils::NbrUnit>& builder, - std::shared_ptr offsets, uint32_t tvnum, int concurrency, + const int64_t* offsets, uint32_t tvnum, int concurrency, bool& is_multigraph); template boost::leaf::result generate_directed_csr( @@ -59,7 +58,7 @@ template boost::leaf::result generate_directed_csr( std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); template boost::leaf::result generate_directed_csc( @@ -68,11 +67,11 @@ template boost::leaf::result generate_directed_csc( std::vector>>>& oedges, - std::vector>& oedge_offsets, + std::vector>& oedge_offsets, std::vector>>>& iedges, - std::vector>& iedge_offsets, + std::vector>& iedge_offsets, bool& is_multigraph); template boost::leaf::result generate_undirected_csr( @@ -83,7 +82,7 @@ template boost::leaf::result generate_undirected_csr( std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); template boost::leaf::result @@ -95,7 +94,7 @@ generate_undirected_csr_memopt( std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); } // namespace vineyard diff --git a/modules/graph/fragment/property_graph_utils_uint64.cc b/modules/graph/fragment/property_graph_utils_uint64.cc index 0f2eebb3..b3ca82ab 100644 --- a/modules/graph/fragment/property_graph_utils_uint64.cc +++ b/modules/graph/fragment/property_graph_utils_uint64.cc @@ -42,13 +42,12 @@ template boost::leaf::result generate_local_id_list( template void sort_edges_with_respect_to_vertex( vineyard::PodArrayBuilder< property_graph_utils::NbrUnit>& builder, - std::shared_ptr offsets, uint64_t tvnum, - int concurrency); + const int64_t* offsets, uint64_t tvnum, int concurrency); template void check_is_multigraph( vineyard::PodArrayBuilder< property_graph_utils::NbrUnit>& builder, - std::shared_ptr offsets, uint64_t tvnum, int concurrency, + const int64_t* offsets, uint64_t tvnum, int concurrency, bool& is_multigraph); template boost::leaf::result generate_directed_csr( @@ -59,7 +58,7 @@ template boost::leaf::result generate_directed_csr( std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); template boost::leaf::result generate_directed_csc( @@ -68,11 +67,11 @@ template boost::leaf::result generate_directed_csc( std::vector>>>& oedges, - std::vector>& oedge_offsets, + std::vector>& oedge_offsets, std::vector>>>& iedges, - std::vector>& iedge_offsets, + std::vector>& iedge_offsets, bool& is_multigraph); template boost::leaf::result generate_undirected_csr( @@ -83,7 +82,7 @@ template boost::leaf::result generate_undirected_csr( std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); template boost::leaf::result @@ -95,7 +94,7 @@ generate_undirected_csr_memopt( std::vector>>>& edges, - std::vector>& edge_offsets, + std::vector>& edge_offsets, bool& is_multigraph); } // namespace vineyard diff --git a/modules/graph/loader/arrow_fragment_loader.cc b/modules/graph/loader/arrow_fragment_loader.cc index f13e87f5..9129d6b3 100644 --- a/modules/graph/loader/arrow_fragment_loader.cc +++ b/modules/graph/loader/arrow_fragment_loader.cc @@ -50,12 +50,12 @@ static Status ReadRecordBatchesFromVineyardStreamImpl( size_t end_to_read = std::min(local_streams.size(), (part_id + 1) * split_size); - VLOG(10) << "reading recordbatches from vineyard: total chunks = " - << local_streams.size() << ", part id = " << part_id - << ", part num = " << part_num - << ", start to read = " << start_to_read - << ", end to read = " << end_to_read - << ", split size = " << split_size; + VLOG(100) << "reading recordbatches from vineyard: total chunks = " + << local_streams.size() << ", part id = " << part_id + << ", part num = " << part_num + << ", start to read = " << start_to_read + << ", end to read = " << end_to_read + << ", split size = " << split_size; std::mutex mutex_for_results; @@ -71,9 +71,7 @@ static Status ReadRecordBatchesFromVineyardStreamImpl( RETURN_ON_ERROR(stream->ReadRecordBatches(read_batches)); { std::lock_guard scoped_lock(mutex_for_results); - for (auto const& batch : read_batches) { - batches.emplace_back(batch); - } + batches.insert(batches.end(), read_batches.begin(), read_batches.end()); } return Status::OK(); }; diff --git a/modules/graph/loader/arrow_fragment_loader_impl.h b/modules/graph/loader/arrow_fragment_loader_impl.h index 5847f86f..79f93053 100644 --- a/modules/graph/loader/arrow_fragment_loader_impl.h +++ b/modules/graph/loader/arrow_fragment_loader_impl.h @@ -313,8 +313,8 @@ ArrowFragmentLoader::loadVertexTables( VY_OK_OR_RAISE(ReadTableFromVineyard(client_, sourceId, table, index, total_parts)); } else { - VY_OK_OR_RAISE(ReadTableFromLocation( - files[label_id] + "#header_row=true", table, index, total_parts)); + VY_OK_OR_RAISE( + ReadTableFromLocation(files[label_id], table, index, total_parts)); } return table; }; diff --git a/src/client/client.cc b/src/client/client.cc index 37494753..a97e2975 100644 --- a/src/client/client.cc +++ b/src/client/client.cc @@ -351,7 +351,8 @@ Status Client::PullNextStreamChunk(ObjectID const id, std::shared_ptr Client::GetObject(const ObjectID id) { ObjectMeta meta; RETURN_NULL_ON_ERROR(this->GetMetaData(id, meta, true)); - RETURN_NULL_ON_ASSERT(!meta.MetaData().empty()); + RETURN_NULL_ON_ASSERT(!meta.MetaData().empty(), + "metadata shouldn't be empty"); auto object = ObjectFactory::Create(meta.GetTypeName()); if (object == nullptr) { object = std::unique_ptr(new Object()); diff --git a/src/client/ds/object_meta.cc b/src/client/ds/object_meta.cc index 31c26718..57b3ef0b 100644 --- a/src/client/ds/object_meta.cc +++ b/src/client/ds/object_meta.cc @@ -173,24 +173,45 @@ std::shared_ptr ObjectMeta::GetMember(const std::string& name) const { return object; } +Status ObjectMeta::GetMember(const std::string& name, + std::shared_ptr& object) const { + ObjectMeta meta; + RETURN_ON_ERROR(GetMemberMeta(name, meta)); + RETURN_ON_ASSERT(!meta.MetaData().empty(), "metadata shouldn't be empty"); + object = ObjectFactory::Create(meta.GetTypeName()); + if (object == nullptr) { + object = std::unique_ptr(new Object()); + } + object->Construct(meta); + return Status::OK(); +} + ObjectMeta ObjectMeta::GetMemberMeta(const std::string& name) const { - ObjectMeta ret; + ObjectMeta meta; + VINEYARD_CHECK_OK(GetMemberMeta(name, meta)); + return meta; +} + +Status ObjectMeta::GetMemberMeta(const std::string& name, + ObjectMeta& meta) const { auto const& child_meta = meta_[name]; - VINEYARD_ASSERT(!child_meta.is_null(), "Failed to get member " + name); + RETURN_ON_ASSERT(!child_meta.is_null(), + "Failed to get member '" + name + "'"); - ret.SetMetaData(this->client_, child_meta); + meta.Reset(); + meta.SetMetaData(this->client_, child_meta); auto const& all_blobs = buffer_set_->AllBuffers(); - for (auto const& blob : ret.buffer_set_->AllBuffers()) { + for (auto const& blob : meta.buffer_set_->AllBuffers()) { auto iter = all_blobs.find(blob.first); // for remote object, the blob may not present here if (iter != all_blobs.end()) { - ret.SetBuffer(blob.first, iter->second); + meta.SetBuffer(blob.first, iter->second); } } if (this->force_local_) { - ret.ForceLocal(); + meta.ForceLocal(); } - return ret; + return Status::OK(); } Status ObjectMeta::GetBuffer(const ObjectID blob_id, diff --git a/src/client/ds/object_meta.h b/src/client/ds/object_meta.h index 44d2d07f..d794f53e 100644 --- a/src/client/ds/object_meta.h +++ b/src/client/ds/object_meta.h @@ -30,6 +30,7 @@ limitations under the License. #include "client/ds/core_types.h" #include "common/util/json.h" #include "common/util/status.h" +#include "common/util/typename.h" #include "common/util/uuid.h" namespace vineyard { @@ -636,14 +637,67 @@ class ObjectMeta { */ std::shared_ptr GetMember(const std::string& name) const; + /** + * @brief Get member value from vineyard. + * + * @param name The name of member object. + * + * @return member The member object. + */ + Status GetMember(const std::string& name, + std::shared_ptr& object) const; + + /** + * @brief Get member value from vineyard. + * + * @param name The name of member object. + * + * @return member The member object. + */ + template + std::shared_ptr GetMember(const std::string& name) const { + return std::dynamic_pointer_cast(GetMember(name)); + } + + /** + * @brief Get member value from vineyard. + * + * @param name The name of member object. + * + * @return member The member object. + */ + template + Status GetMember(const std::string& name, std::shared_ptr& object) const { + std::shared_ptr _object; + RETURN_ON_ERROR(GetMember(name, _object)); + object = std::dynamic_pointer_cast(_object); + if (object == nullptr) { + return Status::ObjectTypeError(type_name(), + GetMemberMeta(name).GetTypeName()); + } else { + return Status::OK(); + } + } + /** * @brief Get member's ObjectMeta value. * * @param name The name of member object. - * @param member The metadata of member object. will be stored in `value`. + * + * @return member The metadata of member object. will be stored in `value`. */ ObjectMeta GetMemberMeta(const std::string& name) const; + /** + * @brief Get member's ObjectMeta value. + * + * @param name The name of member object. + * @param meta The metadata of member object. + * + * @return Whether the member metadata has been found. + */ + Status GetMemberMeta(const std::string& name, ObjectMeta& meta) const; + /** * @brief Get buffer member (directed or indirected) from the metadata. The * metadata should has already been initialized. diff --git a/src/common/util/env.cc b/src/common/util/env.cc index 62716521..9b71d06e 100644 --- a/src/common/util/env.cc +++ b/src/common/util/env.cc @@ -109,7 +109,13 @@ void create_dirs(const char* path) { * c.f.: https://stackoverflow.com/a/14927379/5080177 */ size_t get_rss(bool include_shared_memory) { + // why "trim_rss" first? + // + // - for more accurate statistics + // - as a hint for allocator to release pages in places where `get_rss()` + // is called (where memory information is in cencern) in programs. trim_rss(); + #if defined(__APPLE__) && defined(__MACH__) /* OSX ------------------------------------------------------ */ struct mach_task_basic_info info;