diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ce1327..c88c589 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ include( ${CMAKE_CURRENT_BINARY_DIR}/cmake/get_cpm.cmake ) # Add packages -set( boost_ver boost-1.86.0 ) +set( boost_ver boost-1.87.0 ) CPMAddPackage( "gh:boostorg/static_assert#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/throw_exception#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/config#${boost_ver}" ) # Boost::core dependency diff --git a/include/psi/vm/align.hpp b/include/psi/vm/align.hpp index ca372c8..8721cc9 100644 --- a/include/psi/vm/align.hpp +++ b/include/psi/vm/align.hpp @@ -26,7 +26,7 @@ namespace align_detail [[ using gnu: const, always_inline ]] constexpr auto is_aligned( auto const value, auto const alignment ) noexcept { return __builtin_is_aligned( value, alignment ); } #else [[ using gnu: const, always_inline ]] constexpr auto is_aligned( auto const value, auto const alignment ) noexcept { return value % alignment == 0; } -[[ using gnu: const, always_inline ]] constexpr auto is_aligned( auto * const ptr , auto const alignment ) noexcept { return is_aligned( reinterpret_cast( ptr ), alignment ); } +[[ using gnu: const, always_inline ]] constexpr auto is_aligned( auto * const ptr , auto const alignment ) noexcept { return is_aligned( std::bit_cast( ptr ), alignment ); } #endif [[ using gnu: const, always_inline ]] constexpr auto align_down( auto const value, auto const alignment ) noexcept { diff --git a/include/psi/vm/containers/allocator.hpp b/include/psi/vm/containers/allocator.hpp index fb4aed6..dbc20e0 100644 --- a/include/psi/vm/containers/allocator.hpp +++ b/include/psi/vm/containers/allocator.hpp @@ -26,7 +26,7 @@ namespace psi::vm { //------------------------------------------------------------------------------ -namespace detail { [[ noreturn, gnu::cold ]] inline void throw_bad_alloc() { throw std::bad_alloc(); } } +namespace detail { [[ noreturn, gnu::cold ]] void throw_bad_alloc() PSI_NOEXCEPT_EXCEPT_BADALLOC; } class allocator_backing_mapping { diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 767039f..9ff7a06 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -1,6 +1,6 @@ #pragma once -#include "vector.hpp" +#include "vm_vector.hpp" #include #include @@ -36,18 +36,6 @@ namespace psi::vm PSI_WARNING_DISABLE_PUSH() PSI_WARNING_MSVC_DISABLE( 5030 ) // unrecognized attribute -namespace detail -{ - template - [[ gnu::const ]] auto header_data( std::span const hdr_storage ) noexcept - { - auto const data { align_up( hdr_storage.data() ) }; - auto const remaining_space{ hdr_storage.size() - unsigned( data - hdr_storage.data() ) }; - BOOST_ASSERT( remaining_space >= sizeof( Header ) ); - return std::pair{ reinterpret_cast
( data ), std::span{ data + sizeof( Header ), remaining_space - sizeof( Header ) } }; - } -} // namespace detail - template concept LookupType = transparent_comparator || std::is_same_v; @@ -212,7 +200,7 @@ class bptree_base depth_t depth_{}; }; // struct header - using node_pool = vm::vector; + using node_pool = vm::vm_vector; protected: void swap( bptree_base & other ) noexcept; @@ -372,7 +360,7 @@ class bptree_base [[ nodiscard ]] N & new_node() { return as( new_node() ); } private: - auto header_data() noexcept { return detail::header_data
( nodes_.user_header_data() ); } + auto header_data() noexcept { return vm::header_data
( nodes_.user_header_data() ); } void assign_nodes_to_free_pool( node_slot::value_type starting_node ) noexcept; diff --git a/include/psi/vm/containers/crt_vector.hpp b/include/psi/vm/containers/crt_vector.hpp index a5e9d92..d26b98d 100644 --- a/include/psi/vm/containers/crt_vector.hpp +++ b/include/psi/vm/containers/crt_vector.hpp @@ -20,6 +20,8 @@ #include #include +#include +#include #include @@ -57,7 +59,7 @@ namespace detail { # if defined( _MSC_VER ) if constexpr ( alignment > guaranteed_alignment ) - return _aligned_msize( const_cast( address ) ); + return _aligned_msize( const_cast( address ), alignment, 0 ); # endif return crt_alloc_size( address ); } @@ -390,17 +392,17 @@ class [[ nodiscard, clang::trivial_abi ]] crt_vector public: using base::base; - crt_vector() noexcept : p_array_{ nullptr }, size_{ 0 }, capacity_{ 0 } {} - explicit crt_vector( crt_vector const & other ) + constexpr crt_vector() noexcept : p_array_{ nullptr }, size_{ 0 }, capacity_{ 0 } {} + constexpr explicit crt_vector( crt_vector const & other ) { auto const data{ storage_init( other.size() ) }; try { std::uninitialized_copy_n( other.data(), other.size(), data ); } catch(...) { al::deallocate( data, capacity() ); throw; } } - crt_vector( crt_vector && other ) noexcept : p_array_{ other.p_array_ }, size_{ other.size_ }, capacity_{ other.capacity_ } { other.mark_freed(); } + constexpr crt_vector( crt_vector && other ) noexcept : p_array_{ other.p_array_ }, size_{ other.size_ }, capacity_{ other.capacity_ } { other.mark_freed(); } - crt_vector & operator=( crt_vector const & other ) { *this = crt_vector( other ); } - crt_vector & operator=( crt_vector && other ) noexcept( std::is_nothrow_move_constructible_v ) + constexpr crt_vector & operator=( crt_vector const & other ) { *this = crt_vector( other ); } + constexpr crt_vector & operator=( crt_vector && other ) noexcept( std::is_nothrow_move_constructible_v ) { std::swap( this->p_array_ , other.p_array_ ); std::swap( this->size_ , other.size_ ); @@ -408,7 +410,7 @@ class [[ nodiscard, clang::trivial_abi ]] crt_vector other.free(); return *this; } - ~crt_vector() noexcept { free(); } + constexpr ~crt_vector() noexcept { free(); } [[ nodiscard, gnu::pure ]] size_type size () const noexcept { return size_; } [[ nodiscard, gnu::pure ]] size_type capacity() const noexcept diff --git a/include/psi/vm/containers/static_vector.hpp b/include/psi/vm/containers/static_vector.hpp index 8f1a54f..61a88e7 100644 --- a/include/psi/vm/containers/static_vector.hpp +++ b/include/psi/vm/containers/static_vector.hpp @@ -41,14 +41,14 @@ union noninitialized_array T data[ size ]; }; // noninitialized_array -struct assert_on_overflow { - [[ noreturn ]] static void operator()() noexcept { +struct assert_on_overflow { // VS17.12.3 MSVC still does not support static operator() + [[ noreturn ]] void operator()() const noexcept { BOOST_ASSERT_MSG( false, "Static vector overflow!" ); std::unreachable(); } }; // assert_on_overflow struct throw_on_overflow { - [[ noreturn ]] static void operator()() { detail::throw_out_of_range(); } + [[ noreturn ]] void operator()() const { detail::throw_out_of_range(); } }; // throw_on_overflow template diff --git a/include/psi/vm/containers/vector_impl.hpp b/include/psi/vm/containers/vector_impl.hpp index 2b168fa..feb8de1 100644 --- a/include/psi/vm/containers/vector_impl.hpp +++ b/include/psi/vm/containers/vector_impl.hpp @@ -50,7 +50,15 @@ namespace psi::vm namespace detail { [[ noreturn, gnu::cold ]] void throw_out_of_range(); +#if PSI_MALLOC_OVERCOMMIT != PSI_OVERCOMMIT_Full [[ noreturn, gnu::cold ]] void throw_bad_alloc (); +#else + [[ gnu::cold ]] inline void throw_bad_alloc() noexcept + { + BOOST_ASSERT_MSG( false, "Unexpected allocation failure" ); + std::unreachable(); + } +#endif template constexpr T * mutable_iter( T const * const ptr ) noexcept { return const_cast( ptr ); } @@ -217,7 +225,7 @@ class [[ nodiscard, clang::trivial_abi ]] vector_impl { if constexpr ( std::random_access_iterator ) { - auto const sz{ std::distance( first, last ) }; + auto const sz{ static_cast( std::distance( first, last ) ) }; auto & impl{ initialized_impl( sz, no_init ) }; // STL utility functions handle EH safety - no need to catch to // reset size as Impl/the derived class should not attempt cleanup @@ -335,14 +343,14 @@ class [[ nodiscard, clang::trivial_abi ]] vector_impl //! of the reversed vector. //! Throws: Nothing. //! Complexity: Constant. - [[ nodiscard ]] auto rbegin( this auto & self ) noexcept { return std::make_reverse_iterator( self.begin() ); } + [[ nodiscard ]] auto rbegin( this auto & self ) noexcept { return std::make_reverse_iterator( self.end() ); } [[ nodiscard ]] const_reverse_iterator crbegin( this Impl const & self ) noexcept { return self.rbegin(); } //! Effects: Returns a reverse_iterator pointing to the end //! of the reversed vector. //! Throws: Nothing. //! Complexity: Constant. - [[ nodiscard ]] auto rend( this auto & self ) noexcept { return std::make_reverse_iterator( self.end() ); } + [[ nodiscard ]] auto rend( this auto & self ) noexcept { return std::make_reverse_iterator( self.begin() ); } [[ nodiscard ]] const_reverse_iterator crend( this Impl const & self ) noexcept { return self.rend(); } ////////////////////////////////////////////// @@ -751,16 +759,6 @@ class [[ nodiscard, clang::trivial_abi ]] vector_impl self.free(); } - //! Effects: Returns true if x and y are equal - //! - //! Complexity: Linear to the number of elements in the container. - [[ nodiscard ]] bool operator==( this auto const & self, std::ranges::range auto const & other ) noexcept { return std::equal( self.begin(), self.end(), other.begin(), other.end() ); } - - //! Effects: Returns true if x and y are unequal - //! - //! Complexity: Linear to the number of elements in the container. - [[ nodiscard ]] bool operator!=( this auto const & self, std::ranges::range auto const & other ) noexcept { return !(self == other); } - void swap( this auto & self, auto & other ) noexcept { std::swap( self, other ); } @@ -873,6 +871,13 @@ class [[ nodiscard, clang::trivial_abi ]] vector_impl } }; // class vector_impl + +//! Effects: Returns the result of std::lexicographical_compare_three_way +//! +//! Complexity: Linear to the number of elements in the container. +[[ nodiscard ]] constexpr auto operator<=>( std::ranges::range auto const & left, std::ranges::range auto const & right ) noexcept { return std::lexicographical_compare_three_way( left.begin(), left.end(), right.begin(), right.end() ); } +[[ nodiscard ]] constexpr auto operator== ( std::ranges::range auto const & left, std::ranges::range auto const & right ) noexcept { return std::equal ( left.begin(), left.end(), right.begin(), right.end() ); } + //------------------------------------------------------------------------------ } // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/vector.hpp b/include/psi/vm/containers/vm_vector.hpp similarity index 70% rename from include/psi/vm/containers/vector.hpp rename to include/psi/vm/containers/vm_vector.hpp index fe4f02a..802afcf 100644 --- a/include/psi/vm/containers/vector.hpp +++ b/include/psi/vm/containers/vm_vector.hpp @@ -31,15 +31,23 @@ namespace detail template struct size {}; } // namespace detail -class [[ clang::trivial_abi ]] contiguous_container_storage_base +class [[ clang::trivial_abi ]] contiguous_storage_base { public: + // Mitigation for alignment codegen being sprayed allover at header_data + // call sites - allow it to assume a small, 'good enough for most', + // guaranteed alignment (event at the possible expense of slack space in + // header hierarchies) so that alignment fixups can be skipped for such + // headers. + static std::uint8_t constexpr minimal_subheader_alignment{ alignof( int ) }; + static std::uint8_t constexpr minimal_total_header_size_alignment{ 16 }; + // TODO shallow/COW copy construction, + OSX vm_copy - contiguous_container_storage_base( contiguous_container_storage_base && ) = default; - contiguous_container_storage_base & operator=( contiguous_container_storage_base && ) = default; + contiguous_storage_base( contiguous_storage_base && ) = default; + contiguous_storage_base & operator=( contiguous_storage_base && ) = default; [[ nodiscard, gnu::pure, gnu::assume_aligned( reserve_granularity ) ]] auto * data() noexcept { BOOST_ASSERT_MSG( mapping_, "Paging file not attached" ); return std::assume_aligned( view_.data() ); } - [[ nodiscard, gnu::pure, gnu::assume_aligned( reserve_granularity ) ]] auto * data() const noexcept { return const_cast( *this ).data(); } + [[ nodiscard, gnu::pure, gnu::assume_aligned( reserve_granularity ) ]] auto * data() const noexcept { return const_cast( *this ).data(); } //! Effects: Tries to deallocate the excess of memory created //! with previous allocations. The size of the vector is unchanged @@ -68,12 +76,12 @@ class [[ clang::trivial_abi ]] contiguous_container_storage_base explicit operator bool() const noexcept { return has_attached_storage(); } - void swap( contiguous_container_storage_base & other ) noexcept { std::swap( *this, other ); } + void swap( contiguous_storage_base & other ) noexcept { std::swap( *this, other ); } protected: static bool constexpr storage_zero_initialized{ true }; - constexpr contiguous_container_storage_base() = default; + constexpr contiguous_storage_base() = default; err::fallible_result map_file( auto const * const file_name, std::size_t const header_size, flags::named_object_construction_policy const policy ) noexcept @@ -107,12 +115,12 @@ class [[ clang::trivial_abi ]] contiguous_container_storage_base private: mapped_view view_; mapping mapping_; -}; // contiguous_container_storage_base +}; // contiguous_storage_base template -class [[ clang::trivial_abi ]] contiguous_container_storage +class [[ clang::trivial_abi ]] contiguous_storage : - public contiguous_container_storage_base, + public contiguous_storage_base, private detail::size // Checkout a revision prior to March the 21st 2024 for a version that used // statically sized header sizes. The current approach is more versatile @@ -126,8 +134,8 @@ class [[ clang::trivial_abi ]] contiguous_container_storage using size_holder = detail::size; public: - contiguous_container_storage( ) noexcept requires( headerless ) = default; - explicit contiguous_container_storage( size_type const header_size ) noexcept requires( !headerless ) : size_holder{ header_size } {} + contiguous_storage( ) noexcept requires( headerless ) = default; + explicit contiguous_storage( size_type const header_size ) noexcept requires( !headerless ) : size_holder{ align_up( header_size, minimal_total_header_size_alignment ) } {} static constexpr std::uint8_t header_size() noexcept requires( headerless ) { return 0; } @@ -135,17 +143,18 @@ class [[ clang::trivial_abi ]] contiguous_container_storage { auto const sz{ size_holder::value }; BOOST_ASSUME( sz >= sizeof( sz_t ) ); + BOOST_ASSUME( is_aligned( sz, minimal_total_header_size_alignment ) ); return sz; } - using contiguous_container_storage_base::operator bool; + using contiguous_storage_base::operator bool; - [[ gnu::pure, nodiscard ]] auto header_storage() noexcept { return std::span{ contiguous_container_storage_base::data(), header_size() - size_size }; } - [[ gnu::pure, nodiscard ]] auto header_storage() const noexcept { return std::span{ contiguous_container_storage_base::data(), header_size() - size_size }; } + [[ gnu::pure, nodiscard ]] auto header_storage() noexcept { return std::span{ contiguous_storage_base::data(), header_size() - size_size }; } + [[ gnu::pure, nodiscard ]] auto header_storage() const noexcept { return std::span{ contiguous_storage_base::data(), header_size() - size_size }; } err::fallible_result map_file( auto const * const file_name, flags::named_object_construction_policy const policy ) noexcept { - auto const sz{ contiguous_container_storage_base::map_file( file_name, header_size(), policy )() }; + auto const sz{ contiguous_storage_base::map_file( file_name, header_size(), policy )() }; if ( !sz ) return sz.error(); if constexpr ( !headerless ) @@ -160,7 +169,7 @@ class [[ clang::trivial_abi ]] contiguous_container_storage err::fallible_result map_memory( size_type const size ) noexcept { - auto const sz{ contiguous_container_storage_base::map_memory( size + header_size() )() }; + auto const sz{ contiguous_storage_base::map_memory( size + header_size() )() }; if ( !sz ) return sz.error(); BOOST_ASSERT( *sz == size + header_size() ); @@ -173,16 +182,16 @@ class [[ clang::trivial_abi ]] contiguous_container_storage return err::success; } - auto data() noexcept { return contiguous_container_storage_base::data() + header_size(); } - auto data() const noexcept { return contiguous_container_storage_base::data() + header_size(); } + auto data() noexcept { return contiguous_storage_base::data() + header_size(); } + auto data() const noexcept { return contiguous_storage_base::data() + header_size(); } //! Effects: Returns true if the vector contains no elements. //! Throws: Nothing. //! Complexity: Constant. [[ nodiscard, gnu::pure ]] bool empty() const noexcept { return !has_attached_storage() || !size(); } - [[ nodiscard, gnu::pure ]] auto storage_size() const noexcept { return static_cast( contiguous_container_storage_base::storage_size() ); } - [[ nodiscard, gnu::pure ]] auto mapped_size() const noexcept { return static_cast( contiguous_container_storage_base:: mapped_size() ); } + [[ nodiscard, gnu::pure ]] auto storage_size() const noexcept { return static_cast( contiguous_storage_base::storage_size() ); } + [[ nodiscard, gnu::pure ]] auto mapped_size() const noexcept { return static_cast( contiguous_storage_base:: mapped_size() ); } [[ nodiscard, gnu::pure ]] auto size () const noexcept { if constexpr ( headerless ) return mapped_size(); else return stored_size(); } [[ nodiscard, gnu::pure ]] auto capacity() const noexcept { if constexpr ( headerless ) return storage_size(); else return mapped_size() - header_size(); } @@ -194,12 +203,12 @@ class [[ clang::trivial_abi ]] contiguous_container_storage if constexpr ( headerless ) { if ( target_size > size() ) - contiguous_container_storage_base::grow_to( target_size ); + contiguous_storage_base::grow_to( target_size ); } else { if ( target_size > capacity() ) - contiguous_container_storage_base::grow_to( target_size + header_size() ); + contiguous_storage_base::grow_to( target_size + header_size() ); stored_size() = target_size; } return data(); @@ -207,7 +216,7 @@ class [[ clang::trivial_abi ]] contiguous_container_storage void * shrink_to( size_type const target_size ) noexcept { - auto const data_ptr{ contiguous_container_storage_base::shrink_to( target_size + header_size() ) }; + auto const data_ptr{ contiguous_storage_base::shrink_to( target_size + header_size() ) }; if constexpr ( !headerless ) stored_size() = target_size; return data_ptr; @@ -241,18 +250,18 @@ class [[ clang::trivial_abi ]] contiguous_container_storage void reserve( size_type const new_capacity ) { if constexpr ( headerless ) - contiguous_container_storage_base::reserve( new_capacity ); + contiguous_storage_base::reserve( new_capacity ); else if ( new_capacity > capacity() ) - contiguous_container_storage_base::grow_to( new_capacity + header_size() ); + contiguous_storage_base::grow_to( new_capacity + header_size() ); } void shrink_to_fit() noexcept { if constexpr ( headerless ) - contiguous_container_storage_base::shrink_to_fit(); + contiguous_storage_base::shrink_to_fit(); else - contiguous_container_storage_base::shrink_to( stored_size() + header_size() ); + contiguous_storage_base::shrink_to( stored_size() + header_size() ); } bool has_extra_capacity() const noexcept @@ -265,7 +274,7 @@ class [[ clang::trivial_abi ]] contiguous_container_storage { BOOST_ASSERT_MSG( sz_delta <= ( capacity() - size() ), "Out of preallocated space" ); if constexpr ( headerless ) - contiguous_container_storage_base::expand_view( mapped_size() + sz_delta ); + contiguous_storage_base::expand_view( mapped_size() + sz_delta ); else stored_size() += sz_delta; } @@ -274,7 +283,7 @@ class [[ clang::trivial_abi ]] contiguous_container_storage { BOOST_ASSUME( new_size <= capacity() ); if constexpr ( headerless ) - contiguous_container_storage_base::shrink_mapped_size_to( new_size ); + contiguous_storage_base::shrink_mapped_size_to( new_size ); else stored_size() = new_size; } @@ -283,32 +292,32 @@ class [[ clang::trivial_abi ]] contiguous_container_storage void free() noexcept { if constexpr ( headerless ) - contiguous_container_storage_base::free(); + contiguous_storage_base::free(); else shrink_to( 0 ); } [[ nodiscard, gnu::pure ]] size_type & stored_size() noexcept requires( !headerless ) { - auto const p_size{ contiguous_container_storage_base::data() + header_size() - size_size }; + auto const p_size{ contiguous_storage_base::data() + header_size() - size_size }; BOOST_ASSERT( reinterpret_cast( p_size ) % alignof( size_type ) == 0 ); return *reinterpret_cast( p_size ); } [[ nodiscard, gnu::pure ]] size_type stored_size() const noexcept requires( !headerless ) { - return const_cast( *this ).stored_size(); + return const_cast( *this ).stored_size(); } -}; // contiguous_container_storage +}; // contiguous_storage template -class [[ clang::trivial_abi ]] typed_contiguous_container_storage +class [[ clang::trivial_abi ]] typed_contiguous_storage : - public contiguous_container_storage, - public vector_impl, T, sz_t> + public contiguous_storage, + public vector_impl, T, sz_t> { private: - using base = contiguous_container_storage; - using vec_impl = vector_impl, T, sz_t>; + using base = contiguous_storage; + using vec_impl = vector_impl, T, sz_t>; public: using value_type = T; @@ -321,8 +330,7 @@ class [[ clang::trivial_abi ]] typed_contiguous_container_storage using vec_impl::resize; [[ nodiscard, gnu::pure ]] T * data() noexcept { return to_t_ptr( base::data() ); } - //using vec_impl::data; - [[ nodiscard, gnu::pure ]] T const * data() const noexcept { return const_cast( *this ).data(); } + [[ nodiscard, gnu::pure ]] T const * data() const noexcept { return const_cast( *this ).data(); } //! Effects: Returns the number of the elements contained in the vector. //! @@ -344,10 +352,10 @@ class [[ clang::trivial_abi ]] typed_contiguous_container_storage decltype( auto ) user_header_data() noexcept { return base::header_storage(); } // helper getter for generic code that wishes to do basic manipulation on - // vm::vectors w/o being templated (contiguous_container_storage_base does not + // vm::vectors w/o being templated (contiguous_storage_base does not // publicize functionality that could be used to make it out of sync with the // corresponding vm::vector) - contiguous_container_storage_base & storage_base() noexcept { return *this; } + contiguous_storage_base & storage_base() noexcept { return *this; } private: friend vec_impl; //! Effects: If n is less than or equal to capacity(), this call has no @@ -375,44 +383,81 @@ private: friend vec_impl; return static_cast( rez ); } PSI_WARNING_DISABLE_POP() -}; // class typed_contiguous_container_storage +}; // class typed_contiguous_storage struct header_info { using align_t = std::uint16_t; // fit page_size + static align_t constexpr minimal_subheader_alignment{ contiguous_storage_base::minimal_subheader_alignment }; + constexpr header_info() = default; - constexpr header_info( std::uint32_t const size, align_t const alignment ) noexcept : header_size{ size }, data_extra_alignment{ alignment } {} + constexpr header_info( std::uint32_t const size, align_t const alignment ) noexcept : header_size{ size }, data_extra_alignment{ std::max( alignment, minimal_subheader_alignment ) } {} template - constexpr header_info( std::in_place_type_t, align_t const extra_alignment = 1 ) noexcept : header_info{ sizeof( T ), extra_alignment } {} + constexpr header_info( std::in_place_type_t, align_t const extra_alignment = alignof( T ) ) noexcept : header_info{ sizeof( T ), extra_alignment } {} template - static constexpr header_info make( align_t const extra_alignment = 1 ) noexcept { return { sizeof( T ), extra_alignment }; } + static constexpr header_info make( align_t const extra_alignment = alignof( T ) ) noexcept { return { sizeof( T ), extra_alignment }; } template - header_info add_header() const noexcept // support chained headers (class hierarchies) + constexpr header_info add_header() const noexcept // support chained headers (class hierarchies) { - auto const aligned_size{ align_up( this->header_size, alignof( AdditionalHeader ) ) }; + auto const alignment{ std::max( alignof( AdditionalHeader ), minimal_subheader_alignment ) }; + auto const padded_size{ align_up( this->header_size, alignment ) }; return { - static_cast( aligned_size + sizeof( AdditionalHeader ) ), - this->data_extra_alignment + static_cast( padded_size + sizeof( AdditionalHeader ) ), + std::max( final_alignment(), alignment ) }; } template - header_info with_alignment_for() const noexcept + constexpr header_info with_alignment_for() const noexcept { - BOOST_ASSUME( data_extra_alignment >= 1 ); - return { this->header_size, std::max({ this->data_extra_alignment, alignof( T )... }) }; + return { this->header_size, std::max( { this->final_alignment(), alignof( T )... } ) }; } - std::uint32_t final_header_size() const noexcept { return align_up( header_size, data_extra_alignment ); } + constexpr std::uint32_t final_header_size() const noexcept { return align_up( header_size, final_alignment() ); } + constexpr align_t final_alignment () const noexcept + { + BOOST_ASSUME( data_extra_alignment >= minimal_subheader_alignment ); + auto const is_pow2{ std::has_single_bit( data_extra_alignment ) }; + BOOST_ASSUME( is_pow2 ); + return data_extra_alignment; + } std::uint32_t header_size { 0 }; - align_t data_extra_alignment{ 1 }; // e.g. for vectorization or overlaying complex types over std::byte storage + align_t data_extra_alignment{ minimal_subheader_alignment }; // e.g. for vectorization or overlaying complex types over std::byte storage }; // header_info +// utility function for extracting 'sub-header' data (i.e. intermediate headers +// in an inheritance hierarchy) +template +[[ gnu::const ]] auto header_data( std::span const hdr_storage ) noexcept +{ + auto const in_alignment{ header_info::minimal_subheader_alignment }; + if constexpr ( alignof( Header ) <= in_alignment ) // even with all the assume hints Clang v18 still cannot eliminate redundant fixups so we have to do it explicitly + { + return std::pair + { + reinterpret_cast
( hdr_storage.data() ), + hdr_storage.subspan( align_up( sizeof( Header ) ) ) + }; + } + else + { + auto const raw_data { std::assume_aligned( hdr_storage.data() ) }; + auto const aligned_data { align_up( raw_data ) }; + auto const aligned_space{ static_cast( hdr_storage.size() ) - unsigned( aligned_data - raw_data ) }; + BOOST_ASSUME( aligned_space >= sizeof( Header ) ); + return std::pair + { + reinterpret_cast
( aligned_data ), + std::span{ align_up( aligned_data + sizeof( Header ) ), aligned_space - sizeof( Header ) } + }; + } +} + // Standard-vector-like/compatible _presistent_ container class template // which uses VM/a mapped object for its backing storage. Currently limited // to trivial_abi types. @@ -420,12 +465,12 @@ struct header_info template requires is_trivially_moveable -class [[ clang::trivial_abi ]] vector +class [[ clang::trivial_abi ]] vm_vector : - public typed_contiguous_container_storage + public typed_contiguous_storage { private: - using storage_t = typed_contiguous_container_storage; + using storage_t = typed_contiguous_storage; //using impl = vector_impl; using impl = storage_t; @@ -435,7 +480,7 @@ class [[ clang::trivial_abi ]] vector // not really a standard allocator: providing the alias simply to have boost::container::flat* compileable with this container. using allocator_type = std::allocator; - vector() noexcept requires( headerless ) {} + vm_vector() noexcept requires( headerless ) {} // Allowing for a header to store/persist the 'actual' size of the container can be generalized // to storing arbitrary (types of) headers. This however makes this class template no longer // model just a vector like container but rather a structure consisting of a fixed-sized part @@ -445,7 +490,7 @@ class [[ clang::trivial_abi ]] vector // ability/functionality publicly - as a runtime parameter (instead of a Header template type // parameter) - the relative runtime cost should be near non-existent vs all the standard // benefits (having concrete base classes, less template instantiations and codegen copies...). - explicit vector( header_info const hdr_info = {} ) noexcept requires( !headerless ) + explicit vm_vector( header_info const hdr_info = {} ) noexcept requires( !headerless ) // TODO: use slack space (if any) in the object placed (by user code) in the header space // to store the size (to avoid wasting even more slack space due to alignment padding // after appending the size ('member'). @@ -458,13 +503,13 @@ class [[ clang::trivial_abi ]] vector . final_header_size() ) {} - vector( vector const & ) = delete; - vector( vector && ) = default; + vm_vector( vm_vector const & ) = delete; + vm_vector( vm_vector && ) = default; - ~vector() noexcept = default; + ~vm_vector() noexcept = default; - vector & operator=( vector && ) = default; - vector & operator=( vector const & ) = default; + vm_vector & operator=( vm_vector && ) = default; + vm_vector & operator=( vm_vector const & ) = default; auto map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { BOOST_ASSUME( !storage_t::has_attached_storage() ); return storage_t::map_file( file, policy ); } template diff --git a/src/allocation/allocation.posix.cpp b/src/allocation/allocation.posix.cpp index ff8736c..4a17891 100644 --- a/src/allocation/allocation.posix.cpp +++ b/src/allocation/allocation.posix.cpp @@ -51,8 +51,7 @@ void * mmap( void * const target_address, std::size_t const size, int const prot if ( succeeded ) [[ likely ]] { BOOST_ASSERT( !target_address || ( actual_address == target_address ) || ( ( flags & MAP_FIXED ) == 0 ) ); - BOOST_ASSERT( is_aligned( actual_address, reserve_granularity ) ); - return actual_address; + return std::assume_aligned( actual_address ); } return nullptr; @@ -91,8 +90,8 @@ void decommit( void * const address, std::size_t const size ) noexcept #if 0 // should not be neccessary? BOOST_VERIFY ( - ::madvise( actual_address, newSize, MADV_FREE ) == 0 || - ::madvise( actual_address, newSize, MADV_DONTNEED ) == 0 + ::madvise( actual_address, size, MADV_FREE ) == 0 || + ::madvise( actual_address, size, MADV_DONTNEED ) == 0 ); #endif } diff --git a/src/containers/vector.cpp b/src/containers/vm_vector.cpp similarity index 76% rename from src/containers/vector.cpp rename to src/containers/vm_vector.cpp index 5045c97..1419b71 100644 --- a/src/containers/vector.cpp +++ b/src/containers/vm_vector.cpp @@ -11,10 +11,12 @@ /// //////////////////////////////////////////////////////////////////////////////// //------------------------------------------------------------------------------ -#include +#include #include +#include + #include //------------------------------------------------------------------------------ namespace psi::vm @@ -24,19 +26,21 @@ namespace psi::vm namespace detail { [[ noreturn, gnu::cold ]] void throw_out_of_range() { throw std::out_of_range( "vm::vector access out of bounds" ); } +#if PSI_MALLOC_OVERCOMMIT != PSI_OVERCOMMIT_Full [[ noreturn, gnu::cold ]] void throw_bad_alloc () { throw std::bad_alloc(); } +#endif } // namespace detail -void contiguous_container_storage_base::close() noexcept +void contiguous_storage_base::close() noexcept { mapping_.close(); unmap(); } -void contiguous_container_storage_base::flush_async ( std::size_t const beginning, std::size_t const size ) const noexcept { vm::flush_async ( mapped_span({ view_.subspan( beginning, size ) }) ); } -void contiguous_container_storage_base::flush_blocking( std::size_t const beginning, std::size_t const size ) const noexcept { vm::flush_blocking( mapped_span({ view_.subspan( beginning, size ) }), mapping_.underlying_file() ); } +void contiguous_storage_base::flush_async ( std::size_t const beginning, std::size_t const size ) const noexcept { vm::flush_async ( mapped_span({ view_.subspan( beginning, size ) }) ); } +void contiguous_storage_base::flush_blocking( std::size_t const beginning, std::size_t const size ) const noexcept { vm::flush_blocking( mapped_span({ view_.subspan( beginning, size ) }), mapping_.underlying_file() ); } -void * contiguous_container_storage_base::grow_to( std::size_t const target_size ) +void * contiguous_storage_base::grow_to( std::size_t const target_size ) { BOOST_ASSUME( target_size > mapped_size() ); // basic (1.5x) geometric growth implementation @@ -51,14 +55,14 @@ void * contiguous_container_storage_base::grow_to( std::size_t const target_size return expand_view( target_size ); } -void * contiguous_container_storage_base::expand_view( std::size_t const target_size ) +void * contiguous_storage_base::expand_view( std::size_t const target_size ) { BOOST_ASSERT( get_size( mapping_ ) >= target_size ); view_.expand( target_size, mapping_ ); return data(); } -void * contiguous_container_storage_base::shrink_to( std::size_t const target_size ) noexcept( mapping::views_downsizeable ) +void * contiguous_storage_base::shrink_to( std::size_t const target_size ) noexcept( mapping::views_downsizeable ) { if constexpr ( mapping::views_downsizeable ) { @@ -74,7 +78,7 @@ void * contiguous_container_storage_base::shrink_to( std::size_t const target_si return data(); } -void contiguous_container_storage_base::shrink_mapped_size_to( std::size_t const target_size ) noexcept( mapping::views_downsizeable ) +void contiguous_storage_base::shrink_mapped_size_to( std::size_t const target_size ) noexcept( mapping::views_downsizeable ) { if constexpr ( mapping::views_downsizeable ) { @@ -87,32 +91,33 @@ void contiguous_container_storage_base::shrink_mapped_size_to( std::size_t const } } -void contiguous_container_storage_base::free() noexcept +void contiguous_storage_base::free() noexcept { view_.unmap(); set_size( mapping_, 0 )().assume_succeeded(); } -void contiguous_container_storage_base::shrink_to_fit() noexcept +void contiguous_storage_base::shrink_to_fit() noexcept { set_size( mapping_, mapped_size() )().assume_succeeded(); } -void * contiguous_container_storage_base::resize( std::size_t const target_size ) +void * contiguous_storage_base::resize( std::size_t const target_size ) { if ( target_size > mapped_size() ) return grow_to( target_size ); else return shrink_to( target_size ); } -void contiguous_container_storage_base::reserve( std::size_t const new_capacity ) +void contiguous_storage_base::reserve( std::size_t const new_capacity ) { if ( new_capacity > storage_size() ) set_size( mapping_, new_capacity ); } err::fallible_result -contiguous_container_storage_base::map_file( file_handle && file, std::size_t const header_size ) noexcept +contiguous_storage_base::map_file( file_handle && file, std::size_t const header_size ) noexcept { + BOOST_ASSUME( is_aligned( header_size, minimal_total_header_size_alignment ) ); if ( !file ) return error{}; auto const file_size{ get_size( file ) }; @@ -128,7 +133,7 @@ contiguous_container_storage_base::map_file( file_handle && file, std::size_t co } err::fallible_result -contiguous_container_storage_base::map( file_handle && file, std::size_t const mapping_size ) noexcept +contiguous_storage_base::map( file_handle && file, std::size_t const mapping_size ) noexcept { using ap = flags::access_privileges; using flags = flags::mapping; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ca504e3..b20bc9f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,6 +13,7 @@ endif() set( vm_test_sources "${CMAKE_CURRENT_SOURCE_DIR}/b+tree.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/vector.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/vm_vector.cpp" ) add_executable( vm_unit_tests EXCLUDE_FROM_ALL ${vm_test_sources} ) diff --git a/test/vector.cpp b/test/vector.cpp index 22fcf83..e0d6a1a 100644 --- a/test/vector.cpp +++ b/test/vector.cpp @@ -1,4 +1,6 @@ -#include +// TODO create template tests to test all (three) implementations +#include +#include #include @@ -8,44 +10,175 @@ namespace psi::vm { //------------------------------------------------------------------------------ -TEST( vector, anon_memory_backed ) -{ - psi::vm::vector vec; - vec.map_memory(); - EXPECT_EQ( vec.size(), 0 ); - vec.append_range({ 3.14, 0.14, 0.04 }); - EXPECT_EQ( vec.size(), 3 ); - EXPECT_EQ( vec[ 0 ], 3.14 ); - EXPECT_EQ( vec[ 1 ], 0.14 ); - EXPECT_EQ( vec[ 2 ], 0.04 ); - vec.grow_by( 12345678, default_init ); - // test growth (with 'probable' relocation) does not destroy contents - EXPECT_EQ( vec[ 0 ], 3.14 ); - EXPECT_EQ( vec[ 1 ], 0.14 ); - EXPECT_EQ( vec[ 2 ], 0.04 ); -} - -TEST( vector, file_backed ) -{ - auto const test_vec{ "test.vec" }; - { - psi::vm::vector vec; - vec.map_file( test_vec, flags::named_object_construction_policy::create_new_or_truncate_existing ); - EXPECT_EQ( vec.size(), 0 ); - vec.append_range({ 3.14, 0.14, 0.04 }); - EXPECT_EQ( vec.size(), 3 ); - EXPECT_EQ( vec[ 0 ], 3.14 ); - EXPECT_EQ( vec[ 1 ], 0.14 ); - EXPECT_EQ( vec[ 2 ], 0.04 ); +TEST(vector_test, construction) { + crt_vector vec1; // Default constructor + EXPECT_TRUE(vec1.empty()); + + crt_vector vec2(5, 42); // Constructor with size and value + EXPECT_EQ(vec2.size(), 5); + EXPECT_EQ(vec2[0], 42); + + crt_vector vec3{1, 2, 3, 4, 5}; // Initializer list constructor + EXPECT_EQ(vec3.size(), 5); + EXPECT_EQ(vec3[4], 5); + + crt_vector vec4(vec3.begin(), vec3.end()); // Range constructor + EXPECT_EQ(vec4, vec3); +} + + +TEST(vector_test, element_access) { + crt_vector vec{10, 20, 30, 40}; + + EXPECT_EQ(vec[2], 30); // operator[] + EXPECT_EQ(vec.at(3), 40); // .at() +#if defined( __APPLE__ /*libunwind: malformed __unwind_info at 0x102558A6C bad second level page*/ ) || ( defined( __linux__ ) && !defined( NDEBUG ) /*dubious asan new-free and exception type mismatch*/ ) + // TODO :wat: +#else + EXPECT_THROW( std::ignore = vec.at( 10 ), std::out_of_range ); // .at() with invalid index +#endif + + EXPECT_EQ(vec.front(), 10); // .front() + EXPECT_EQ(vec.back(), 40); // .back() +} + + +TEST(vector_test, modifiers) { + crt_vector vec; + + // Test push_back + vec.push_back(1); + vec.push_back(2); + EXPECT_EQ(vec.size(), 2); + EXPECT_EQ(vec[1], 2); + + // Test emplace_back + vec.emplace_back(3); + EXPECT_EQ(vec.size(), 3); + EXPECT_EQ(vec[2], 3); + + // Test pop_back + vec.pop_back(); + EXPECT_EQ(vec.size(), 2); + EXPECT_EQ(vec.back(), 2); + + // Test insert + vec.insert(vec.begin(), 0); + EXPECT_EQ(vec.front(), 0); + EXPECT_EQ(vec.size(), 3); + + // Test erase + vec.erase(vec.begin()); + EXPECT_EQ(vec.front(), 1); + EXPECT_EQ(vec.size(), 2); + + // Test clear + vec.clear(); + EXPECT_TRUE(vec.empty()); +} + + +TEST(vector_test, capacity) { + crt_vector vec; + EXPECT_TRUE(vec.empty()); + + vec.resize(10, 42); // Resize to larger size + EXPECT_EQ(vec.size(), 10); + EXPECT_EQ(vec[5], 42); + + vec.resize(5); // Resize to smaller size + EXPECT_EQ(vec.size(), 5); + + vec.shrink_to_fit(); // Shrink to fit (no testable behavior, but call it) + EXPECT_GE(vec.capacity(), vec.size()); +} + + +TEST(vector_test, range_support) { + auto range = std::views::iota(1, 6); // Range [1, 2, 3, 4, 5] + crt_vector vec(range.begin(), range.end()); + EXPECT_EQ(vec.size(), 5); + EXPECT_EQ(vec[0], 1); + EXPECT_EQ(vec[4], 5); + vec.append_range({ 321, 654, 78, 0, 9 }); + EXPECT_EQ(vec[7], 78); +} + + +TEST(vector_test, move_semantics) { + crt_vector vec1{1, 2, 3, 4, 5}; + crt_vector vec2(std::move(vec1)); // Move constructor + + EXPECT_TRUE(vec1.empty()); + EXPECT_EQ(vec2.size(), 5); + EXPECT_EQ(vec2[0], 1); + + crt_vector vec3; + vec3 = std::move(vec2); // Move assignment + EXPECT_TRUE(vec2.empty()); + EXPECT_EQ(vec3.size(), 5); + EXPECT_EQ(vec3[0], 1); +} + + +TEST(vector_test, iterators) { + crt_vector vec{10, 20, 30, 40}; + + // Validate iterator traversal + int sum = 0; + for (auto it = vec.begin(); it != vec.end(); ++it) { + sum += *it; } - { - psi::vm::vector vec; - vec.map_file( test_vec, flags::named_object_construction_policy::open_existing ); - EXPECT_EQ( vec.size(), 3 ); - EXPECT_EQ( vec[ 0 ], 3.14 ); - EXPECT_EQ( vec[ 1 ], 0.14 ); - EXPECT_EQ( vec[ 2 ], 0.04 ); + EXPECT_EQ(sum, 100); + + // Test reverse iterators + crt_vector reversed(vec.rbegin(), vec.rend()); + EXPECT_EQ(reversed[0], 40); + EXPECT_EQ(reversed[3], 10); + + // Const iterators + crt_vector const const_vec{ 1, 2, 3 }; + EXPECT_EQ(*const_vec.cbegin(), 1); +} + + +TEST(vector_test, edge_cases) { + crt_vector vec1; + + // Large vector test (memory constraints permitting) +#if 0 // TODO optional size_type overflow handling + try { + crt_vector vec2(std::numeric_limits::max() / 2); + ADD_FAILURE() << "Expected bad_alloc exception for large vector"; + } catch ( std::bad_alloc const & ) { + SUCCEED(); + } catch (...) { + ADD_FAILURE() << "Unexpected exception type for large vector"; } +#endif + + // Test with non-trivial types + struct non_trivial { + int value; + non_trivial(int v) : value(v) {} + ~non_trivial() { value = -1; } + }; + + static_vector vec3; + vec3.emplace_back(42); + EXPECT_EQ(vec3[0].value, 42); +} + + +TEST(vector_test, comparison) { + crt_vector vec1{1, 2, 3}; + crt_vector vec2{1, 2, 3}; + crt_vector vec3{1, 2, 4}; + + EXPECT_TRUE (vec1 == vec2); + EXPECT_TRUE (vec1 != vec3); + EXPECT_TRUE (vec1 < vec3); + EXPECT_FALSE(vec3 < vec1); } //------------------------------------------------------------------------------ diff --git a/test/vm_vector.cpp b/test/vm_vector.cpp new file mode 100644 index 0000000..8cebea3 --- /dev/null +++ b/test/vm_vector.cpp @@ -0,0 +1,53 @@ +#include + +#include + +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +TEST( vm_vector, anon_memory_backed ) +{ + psi::vm::vm_vector vec; + vec.map_memory(); + EXPECT_EQ( vec.size(), 0 ); + vec.append_range({ 3.14, 0.14, 0.04 }); + EXPECT_EQ( vec.size(), 3 ); + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); + vec.grow_by( 12345678, default_init ); + // test growth (with 'probable' relocation) does not destroy contents + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); +} + +TEST( vm_vector, file_backed ) +{ + auto const test_vec{ "test.vec" }; + { + psi::vm::vm_vector vec; + vec.map_file( test_vec, flags::named_object_construction_policy::create_new_or_truncate_existing ); + EXPECT_EQ( vec.size(), 0 ); + vec.append_range({ 3.14, 0.14, 0.04 }); + EXPECT_EQ( vec.size(), 3 ); + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); + } + { + psi::vm::vm_vector vec; + vec.map_file( test_vec, flags::named_object_construction_policy::open_existing ); + EXPECT_EQ( vec.size(), 3 ); + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); + } +} + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------