From 454dacd65813d8e9396c6899a870fecaea70ca84 Mon Sep 17 00:00:00 2001 From: KRM7 <70973547+KRM7@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:49:42 +0100 Subject: [PATCH] layout optimizations for polymorphic types --- README.md | 4 +- src/small_unique_ptr.hpp | 163 +++++++++++++++++++++++--------------- test/small_unique_ptr.cpp | 29 ++++++- 3 files changed, 127 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index bc47669..ae155ba 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ The size of a `small_unique_ptr` object is: The interface matches `std::unique_ptr`, except for: - - There is no deleter template parameter and no associated methods + - There is no `Deleter` template parameter or any of the associated methods - Constructors from pointers are not provided except for the `nullptr` constructor - `release()` is not implemented - - The type of T can't be an array type or `void` + - `T` can't be an incomplete type or an array type - There are a couple of extra methods for checking where objects are allocated The stack buffer is not used in constant evaluated contexts, so any constexpr usage diff --git a/src/small_unique_ptr.hpp b/src/small_unique_ptr.hpp index 62ff889..e065d39 100644 --- a/src/small_unique_ptr.hpp +++ b/src/small_unique_ptr.hpp @@ -9,21 +9,34 @@ #include #include #include +#include #include #include #include namespace detail { - struct empty_t {}; + template + constexpr const T& min(const T& left, const T& right) { return (left < right) ? left : right; } + + template + constexpr const T& max(const T& left, const T& right) { return (left < right) ? right : left; } template - constexpr const T& min(const T& left, const T& right) + void move_buffer(void* src, void* dst) noexcept(std::is_nothrow_move_constructible_v) { - return (left < right) ? left : right; + static_assert(!std::is_const_v && !std::is_volatile_v); + std::construct_at(static_cast(dst), std::move(*static_cast(src))); } + template + concept has_virtual_move = requires(std::remove_cv_t object, void* const dst) + { + requires std::is_polymorphic_v; + { object.small_unique_ptr_move(dst) } noexcept -> std::same_as; + }; + template struct is_nothrow_dereferenceable : std::bool_constant())> {}; @@ -42,15 +55,15 @@ namespace detail inline constexpr bool is_proper_base_of_v = is_proper_base_of::value; - inline constexpr std::size_t cache_line_size = 64; + inline constexpr std::size_t small_ptr_size = detail::max(64, std::hardware_destructive_interference_size); template struct buffer_size { private: - static constexpr std::size_t dynamic_buffer_size = cache_line_size - sizeof(void*) - sizeof(void(*)()); - static constexpr std::size_t static_buffer_size = detail::min(sizeof(T), cache_line_size - sizeof(void*)); + static constexpr std::size_t dynamic_buffer_size = small_ptr_size - sizeof(T*) - !has_virtual_move * sizeof(decltype(&move_buffer)); + static constexpr std::size_t static_buffer_size = detail::min(sizeof(T), small_ptr_size - sizeof(T*)); public: static constexpr std::size_t value = std::has_virtual_destructor_v ? dynamic_buffer_size : static_buffer_size; }; @@ -63,8 +76,8 @@ namespace detail struct buffer_alignment { private: - static constexpr std::size_t dynamic_buffer_alignment = cache_line_size; - static constexpr std::size_t static_buffer_alignment = detail::min(alignof(T), cache_line_size); + static constexpr std::size_t dynamic_buffer_alignment = small_ptr_size; + static constexpr std::size_t static_buffer_alignment = detail::max(alignof(T*), detail::min(alignof(T), small_ptr_size)); public: static constexpr std::size_t value = std::has_virtual_destructor_v ? dynamic_buffer_alignment : static_buffer_alignment; }; @@ -74,95 +87,112 @@ namespace detail template - struct always_heap_allocated + struct is_always_heap_allocated { static constexpr bool value = (sizeof(T) > buffer_size_v) || (alignof(T) > buffer_alignment_v) || (!std::is_abstract_v && !std::is_nothrow_move_constructible_v); }; template - inline constexpr bool always_heap_allocated_v = always_heap_allocated::value; - + inline constexpr bool is_always_heap_allocated_v = is_always_heap_allocated::value; - template - void move_buffer(void* src, void* dst) noexcept - { - std::construct_at(static_cast(dst), std::move(*static_cast(src))); - } template struct small_unique_ptr_base { - using pointer = std::remove_cv_t*; - using const_pointer = const std::remove_cv_t*; - - using buffer_t = unsigned char[buffer_size_v]; - using move_fn = void(*)(void* src, void* dst) noexcept; - - pointer buffer(std::ptrdiff_t offset = 0) noexcept - { - return std::launder(reinterpret_cast(std::addressof(buffer_[0]) + offset)); - } + using pointer = std::remove_cv_t*; + using buffer_t = unsigned char[buffer_size_v]; + using move_fn = void(*)(void* src, void* dst) noexcept; - const_pointer buffer(std::ptrdiff_t offset = 0) const noexcept + pointer buffer(std::ptrdiff_t offset = 0) const noexcept { - return std::launder(reinterpret_cast(std::addressof(buffer_[0]) + offset)); + assert(0 <= offset && offset < buffer_size_v); + return std::launder(reinterpret_cast(std::addressof(buffer_[offset]))); } template void move_buffer_to(small_unique_ptr_base& dst) noexcept { - move_(buffer(), dst.buffer()); + move_(this->buffer(), dst.buffer()); dst.move_ = move_; } constexpr bool is_stack_allocated() const noexcept { return static_cast(this->move_); } - alignas(buffer_alignment_v) - mutable buffer_t buffer_ = {}; + alignas(buffer_alignment_v) mutable buffer_t buffer_ = {}; T* data_ = nullptr; move_fn move_ = nullptr; }; template - requires(!std::is_polymorphic_v && !always_heap_allocated_v) + requires(is_always_heap_allocated_v) struct small_unique_ptr_base { - using pointer = std::remove_cv_t*; - using const_pointer = const std::remove_cv_t*; + static constexpr bool is_stack_allocated() noexcept { return false; } - using buffer_t = unsigned char[buffer_size_v]; - using move_fn = void(*)(void* src, void* dst) noexcept; + T* data_ = nullptr; + }; - pointer buffer(std::ptrdiff_t offset = 0) noexcept - { - return std::launder(reinterpret_cast(std::addressof(buffer_[0]) + offset)); - } + template + requires(!is_always_heap_allocated_v && !std::is_polymorphic_v) + struct small_unique_ptr_base + { + using pointer = std::remove_cv_t*; + using buffer_t = unsigned char[buffer_size_v]; - const_pointer buffer(std::ptrdiff_t offset = 0) const noexcept + pointer buffer(std::ptrdiff_t offset = 0) const noexcept { - return std::launder(reinterpret_cast(std::addressof(buffer_[0]) + offset)); + assert(0 <= offset && offset < buffer_size_v); + return std::launder(reinterpret_cast(std::addressof(buffer_[offset]))); } template void move_buffer_to(small_unique_ptr_base& dst) noexcept { - std::construct_at(dst.buffer(), std::move(*buffer())); + std::construct_at(dst.buffer(), std::move(*this->buffer())); } - constexpr bool is_stack_allocated() const noexcept { return !std::is_constant_evaluated() && (data_ == buffer()); } + constexpr bool is_stack_allocated() const noexcept { return !std::is_constant_evaluated() && (data_ == this->buffer()); } - alignas(buffer_alignment_v) - mutable buffer_t buffer_ = {}; + alignas(buffer_alignment_v) mutable buffer_t buffer_ = {}; T* data_ = nullptr; }; template - requires(always_heap_allocated_v) + requires(!is_always_heap_allocated_v && has_virtual_move) struct small_unique_ptr_base { - static constexpr bool is_stack_allocated() noexcept { return false; } + using pointer = std::remove_cv_t*; + using buffer_t = unsigned char[buffer_size_v]; + pointer buffer(std::ptrdiff_t offset = 0) const noexcept + { + assert(0 <= offset && offset < buffer_size_v); + return std::launder(reinterpret_cast(std::addressof(buffer_[offset]))); + } + + template + requires(has_virtual_move) + void move_buffer_to(small_unique_ptr_base& dst) noexcept + { + const pointer data = const_cast(data_); + data->small_unique_ptr_move(dst.buffer()); + } + + constexpr bool is_stack_allocated() const noexcept + { + if (std::is_constant_evaluated()) return false; + + const volatile unsigned char* data = reinterpret_cast(data_); + const volatile unsigned char* buffer_first = std::addressof(this->buffer_[0]); + const volatile unsigned char* buffer_last = std::addressof(this->buffer_[buffer_size_v]); + + assert(reinterpret_cast(buffer_last) - reinterpret_cast(buffer_first) == buffer_size_v); + + return std::less_equal{}(buffer_first, data) && std::less{}(data, buffer_last); + } + + alignas(buffer_alignment_v) mutable buffer_t buffer_ = {}; T* data_ = nullptr; }; @@ -179,15 +209,17 @@ class small_unique_ptr : private detail::small_unique_ptr_base using pointer = T*; using reference = T&; + struct constructor_tag_t {}; + constexpr small_unique_ptr() noexcept = default; constexpr small_unique_ptr(std::nullptr_t) noexcept {} constexpr small_unique_ptr(small_unique_ptr&& other) noexcept : - small_unique_ptr(std::move(other), detail::empty_t{}) + small_unique_ptr(std::move(other), constructor_tag_t{}) {} template - constexpr small_unique_ptr(small_unique_ptr&& other, detail::empty_t = {}) noexcept + constexpr small_unique_ptr(small_unique_ptr&& other, constructor_tag_t = {}) noexcept { static_assert(!detail::is_proper_base_of_v || std::has_virtual_destructor_v); @@ -196,7 +228,7 @@ class small_unique_ptr : private detail::small_unique_ptr_base this->data_ = std::exchange(other.data_, nullptr); return; } - if constexpr (!detail::always_heap_allocated_v) // other.is_stack_allocated() + if constexpr (!detail::is_always_heap_allocated_v) // other.is_stack_allocated() { other.move_buffer_to(*this); this->data_ = this->buffer(other.template offsetof_base()); @@ -221,7 +253,7 @@ class small_unique_ptr : private detail::small_unique_ptr_base reset(std::exchange(other.data_, nullptr)); return *this; } - if constexpr (!detail::always_heap_allocated_v) // other.is_stack_allocated() + if constexpr (!detail::is_always_heap_allocated_v) // other.is_stack_allocated() { reset(); other.move_buffer_to(*this); @@ -239,22 +271,22 @@ class small_unique_ptr : private detail::small_unique_ptr_base constexpr ~small_unique_ptr() noexcept { - reset(); + is_stack_allocated() ? std::destroy_at(this->data_) : delete this->data_; } constexpr void reset(pointer new_data = pointer{}) noexcept { is_stack_allocated() ? std::destroy_at(this->data_) : delete this->data_; + if constexpr (requires { small_unique_ptr::move_; }) this->move_ = nullptr; this->data_ = new_data; - if constexpr (!detail::always_heap_allocated_v && std::is_polymorphic_v) this->move_ = nullptr; } - constexpr void swap(small_unique_ptr& other) noexcept requires(detail::always_heap_allocated_v) + constexpr void swap(small_unique_ptr& other) noexcept requires(detail::is_always_heap_allocated_v) { std::swap(this->data_, other.data_); } - constexpr void swap(small_unique_ptr& other) noexcept requires(!detail::always_heap_allocated_v) + constexpr void swap(small_unique_ptr& other) noexcept requires(!detail::is_always_heap_allocated_v) { if (is_stack_allocated() && other.is_stack_allocated()) { @@ -264,11 +296,12 @@ class small_unique_ptr : private detail::small_unique_ptr_base detail::small_unique_ptr_base temp; other.move_buffer_to(temp); + temp.data_ = temp.buffer(other_offset); std::destroy_at(other.data_); this->move_buffer_to(other); std::destroy_at(this->data_); temp.move_buffer_to(*this); - std::destroy_at(temp.buffer(other_offset)); + std::destroy_at(temp.data_); this->data_ = this->buffer(other_offset); other.data_ = other.buffer(this_offset); @@ -279,13 +312,13 @@ class small_unique_ptr : private detail::small_unique_ptr_base } else if (!is_stack_allocated() && other.is_stack_allocated()) { - T* new_data = this->buffer(other.offsetof_base()); + const pointer new_data = this->buffer(other.offsetof_base()); other.move_buffer_to(*this); other.reset(std::exchange(this->data_, new_data)); } else /* if (is_stack_allocated() && !other.is_stack_allocated()) */ { - T* new_data = other.buffer(this->offsetof_base()); + const pointer new_data = other.buffer(this->offsetof_base()); this->move_buffer_to(other); reset(std::exchange(other.data_, new_data)); } @@ -294,7 +327,7 @@ class small_unique_ptr : private detail::small_unique_ptr_base [[nodiscard]] constexpr static bool is_always_heap_allocated() noexcept { - return detail::always_heap_allocated_v; + return detail::is_always_heap_allocated_v; } [[nodiscard]] @@ -400,16 +433,16 @@ template { small_unique_ptr ptr; - if (detail::always_heap_allocated_v || std::is_constant_evaluated()) + if (detail::is_always_heap_allocated_v || std::is_constant_evaluated()) { ptr.data_ = new T(std::forward(args)...); } - else if constexpr (!detail::always_heap_allocated_v && std::is_polymorphic_v) + else if constexpr (!detail::is_always_heap_allocated_v && std::is_polymorphic_v && !detail::has_virtual_move) { ptr.data_ = std::construct_at(ptr.buffer(), std::forward(args)...); - ptr.move_ = detail::move_buffer; + ptr.move_ = detail::move_buffer>; } - else if constexpr (!detail::always_heap_allocated_v && !std::is_polymorphic_v) + else if constexpr (!detail::is_always_heap_allocated_v) { ptr.data_ = std::construct_at(ptr.buffer(), std::forward(args)...); } @@ -417,4 +450,4 @@ template return ptr; } -#endif // !SMALL_UNIQUE_PTR_HPP \ No newline at end of file +#endif // !SMALL_UNIQUE_PTR_HPP diff --git a/test/small_unique_ptr.cpp b/test/small_unique_ptr.cpp index ea74c58..4aff134 100644 --- a/test/small_unique_ptr.cpp +++ b/test/small_unique_ptr.cpp @@ -13,6 +13,11 @@ struct Base constexpr virtual void value(int) {} constexpr virtual int padding() const { return 0; } constexpr virtual ~Base() noexcept {}; + + virtual void small_unique_ptr_move(void* dst) noexcept + { + std::construct_at(static_cast(dst), std::move(*this)); + } }; template @@ -25,6 +30,12 @@ struct Derived : Base constexpr int value() const override { return value_; } constexpr void value(int n) override { value_ = n; } constexpr int padding() const override { return Padding; } + + void small_unique_ptr_move(void* dst) noexcept override + { + std::construct_at(static_cast(dst), std::move(*this)); + } + private: unsigned char padding_[Padding] = {}; int value_ = Padding; @@ -37,12 +48,22 @@ struct SmallPOD { char dummy_; }; struct LargePOD { char dummy_[128]; }; +TEST_CASE("traits", "[small_unique_ptr]") +{ + STATIC_REQUIRE(std::is_standard_layout_v>); + STATIC_REQUIRE(std::is_standard_layout_v>); + STATIC_REQUIRE(std::is_standard_layout_v>); + + STATIC_REQUIRE(std::is_standard_layout_v>); + STATIC_REQUIRE(std::is_standard_layout_v>); +} + TEST_CASE("object_size", "[small_unique_ptr]") { - STATIC_REQUIRE(sizeof(small_unique_ptr) == detail::cache_line_size); + STATIC_REQUIRE(sizeof(small_unique_ptr) == detail::small_ptr_size); STATIC_REQUIRE(sizeof(small_unique_ptr) == sizeof(void*)); - STATIC_REQUIRE(alignof(small_unique_ptr) == detail::cache_line_size); + STATIC_REQUIRE(alignof(small_unique_ptr) == detail::small_ptr_size); STATIC_REQUIRE(alignof(small_unique_ptr) == alignof(void*)); STATIC_REQUIRE(sizeof(small_unique_ptr) <= 2 * sizeof(void*)); @@ -67,8 +88,12 @@ TEST_CASE("construction", "[small_unique_ptr]") REQUIRE_NOTHROW(make_unique_small()); REQUIRE_NOTHROW(make_unique_small()); REQUIRE_NOTHROW(make_unique_small()); + REQUIRE_NOTHROW(make_unique_small()); REQUIRE_NOTHROW(make_unique_small()); + + REQUIRE_NOTHROW(make_unique_small()); + REQUIRE_NOTHROW(make_unique_small()); } TEST_CASE("is_always_heap_allocated", "[small_unique_ptr]")