diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index e71cc6f..c1abace 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -52,7 +52,7 @@ jobs: submodules: true - uses: actions/setup-java@v3 with: - java-version: '17' + java-version: '23' distribution: 'temurin' - name: Setup latest Xcode on MacOS @@ -60,8 +60,22 @@ jobs: uses: maxim-lobanov/setup-xcode@v1 with: xcode-version: latest + + # For x64 OSX this action offers only v15 https://github.com/KyleMayes/install-llvm-action/blob/master/assets.json + # luckily latest is now ARM by default: + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + - name: Install LLVM and Clang on MacOS + if: matrix.os == 'macos-latest' + uses: KyleMayes/install-llvm-action@v2.0.5 + with: + # https://github.com/KyleMayes/install-llvm-action/issues/65 + arch: arm64 + version: 18 + env: true + - name: Install LLVM and Clang on Linux if: matrix.os == 'ubuntu-latest' + # https://apt.llvm.org run: | wget https://apt.llvm.org/llvm.sh chmod u+x llvm.sh @@ -86,9 +100,11 @@ jobs: # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type run: > cmake -B ${{ steps.strings.outputs.build-output-dir }} + ${{ matrix.cmake_toolchain }} -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }} -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DGitHubCIRun=true -S ${{ github.workspace }} - name: Build diff --git a/include/psi/vm/containers/allocator.hpp b/include/psi/vm/containers/allocator.hpp index 2762b99..fb4aed6 100644 --- a/include/psi/vm/containers/allocator.hpp +++ b/include/psi/vm/containers/allocator.hpp @@ -26,14 +26,14 @@ namespace psi::vm { //------------------------------------------------------------------------------ -namespace detail { [[ noreturn ]] inline void throw_bad_alloc() { throw std::bad_alloc(); } } +namespace detail { [[ noreturn, gnu::cold ]] inline void throw_bad_alloc() { throw std::bad_alloc(); } } class allocator_backing_mapping { public: constexpr allocator_backing_mapping() = default; - err::fallible_result< std::size_t, error > open( auto const * const file_name ) noexcept { return open( create_file( file_name, create_rw_file_flags() ) ); } + err::fallible_result open( auto const * const file_name ) noexcept { return open( create_file( file_name, create_rw_file_flags() ) ); } auto data() const noexcept { return view_.data(); } diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 5f034b8..767039f 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -163,13 +163,15 @@ class bptree_base [[ gnu::pure ]] bool is_root() const noexcept { return !parent; } +# ifndef __clang__ // https://github.com/llvm/llvm-project/issues/36032 protected: // merely to prevent slicing (in return-node-by-ref cases) - constexpr node_header( node_header const & ) noexcept = default; - constexpr node_header( node_header && ) noexcept = default; + constexpr node_header ( node_header const & ) noexcept = default; + constexpr node_header ( node_header && ) noexcept = default; public: - constexpr node_header( ) noexcept = default; + constexpr node_header ( ) noexcept = default; constexpr node_header & operator=( node_header && ) noexcept = default; constexpr node_header & operator=( node_header const & ) noexcept = default; +# endif }; // struct node_header using node_size_type = node_header::size_type; @@ -1346,7 +1348,7 @@ class bptree_base_wkey : public bptree_base { BOOST_ASSUME( target.num_vals + source.num_vals <= target.max_values ); - std::ranges::move( keys( source ), keys( target ).end() ); + std::ranges::move( keys( source ), &target.keys[ target.num_vals ] ); target.num_vals += source.num_vals; source.num_vals = 0; @@ -1387,8 +1389,9 @@ class bptree_base_wkey : public bptree_base move_chldrn( right, 0, num_chldrn( right ), left, num_chldrn( left ) ); auto & separator_key{ parent.keys[ parent_key_idx ] }; left.num_vals += 1; - keys( left ).back() = std::move( separator_key ); - std::ranges::move( keys( right ), keys( left ).end() ); + auto & last_left_key{ keys( left ).back() }; + last_left_key = std::move( separator_key ); + std::ranges::move( keys( right ), std::next( &last_left_key ) ); left.num_vals += right.num_vals; BOOST_ASSUME( left.num_vals >= left.max_values - 1 ); BOOST_ASSUME( left.num_vals <= left.max_values ); diff --git a/include/psi/vm/containers/crt_vector.hpp b/include/psi/vm/containers/crt_vector.hpp new file mode 100644 index 0000000..a5e9d92 --- /dev/null +++ b/include/psi/vm/containers/crt_vector.hpp @@ -0,0 +1,535 @@ +//////////////////////////////////////////////////////////////////////////////// +/// Zero bloat implementation of a classic std::vector around the CRT and/or OS +/// allocation APIs designed for trivially moveable types (eliminating the copy- +/// on-resize overhead of std::vector) + vector_impl extensions. +//////////////////////////////////////////////////////////////////////////////// +/// +/// Copyright (c) Domagoj Saric. +/// +/// Use, modification and distribution is subject to the +/// Boost Software License, Version 1.0. +/// (See accompanying file LICENSE_1_0.txt or copy at +/// http://www.boost.org/LICENSE_1_0.txt) +/// +/// For more information, see http://www.boost.org +/// +//////////////////////////////////////////////////////////////////////////////// +//------------------------------------------------------------------------------ +#pragma once + +#include + +#include + +#include + +#ifdef __linux__ +#include +#elif defined( __APPLE__ ) +#include +#endif +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +namespace detail +{ + // https://www.gnu.org/software/libc/manual/html_node/Aligned-Memory-Blocks.html + inline std::uint8_t constexpr guaranteed_alignment{ 16 }; // all known x64 and arm64 platforms + + [[ gnu::pure ]] inline std::size_t crt_alloc_size( void const * const address ) noexcept + { + // https://lemire.me/blog/2017/09/15/how-fast-are-malloc_size-and-malloc_usable_size-in-c +# if defined( _MSC_VER ) + // https://masm32.com/board/index.php?topic=7018.0 HeapAlloc vs HeapReAlloc + return _msize( const_cast( address ) ); // uber slow & almost useless as it returns the requested alloc size, not the capacity of the allocated block (it just calls HeapSize) +# elif defined( __linux__ ) + return ::malloc_usable_size( const_cast( address ) ); // fast +# elif defined( __APPLE__ ) + return ::malloc_size( address ); // not so fast +# else + static_assert( false, "no malloc size implementation" ); +# endif + } + template + std::size_t crt_aligned_alloc_size( void const * const address ) noexcept + { +# if defined( _MSC_VER ) + if constexpr ( alignment > guaranteed_alignment ) + return _aligned_msize( const_cast( address ) ); +# endif + return crt_alloc_size( address ); + } + + [[ using gnu: assume_aligned( guaranteed_alignment ), malloc, returns_nonnull ]] +#ifdef _MSC_VER + __declspec( noalias, restrict ) +#endif + inline void * crt_realloc( void * const existing_allocation_address, std::size_t const new_size ) + { + auto const new_allocation{ std::realloc( existing_allocation_address, new_size ) }; + if ( !new_allocation ) [[ unlikely ]] + throw_bad_alloc(); + return new_allocation; + } + + [[ using gnu: assume_aligned( guaranteed_alignment ), malloc, returns_nonnull ]] +#ifdef _MSC_VER + __declspec( noalias, restrict ) +#endif + inline void * crt_aligned_realloc( void * const existing_allocation_address, std::size_t const existing_allocation_size, std::size_t const new_size, std::uint8_t const alignment ) + { + BOOST_ASSUME( alignment > guaranteed_alignment ); +# if defined( _MSC_VER ) + std::ignore = existing_allocation_size; + auto const new_allocation{ ::_aligned_realloc( existing_allocation_address, new_size, alignment ) }; +# else + void * new_allocation{ nullptr }; + if ( existing_allocation_address ) [[ likely ]] + { + auto const try_realloc{ std::realloc( existing_allocation_address, new_size ) }; + if ( is_aligned( try_realloc, alignment ) ) [[ likely ]] + { + new_allocation = try_realloc; + } + else + { + BOOST_ASSUME( try_realloc ); // nullptr handled implicitly above + if ( posix_memalign( &new_allocation, alignment, new_size ) == 0 ) [[ likely ]] + std::memcpy( new_allocation, try_realloc, existing_allocation_size ); + std::free( try_realloc ); + } + } + else + { + BOOST_ASSUME( !existing_allocation_size ); + // "On Linux (and other systems), posix_memalign() does not modify + // memptr on failure. A requirement standardizing this behavior was + // added in POSIX .1 - 2008 TC2." + BOOST_VERIFY( posix_memalign( &new_allocation, alignment, new_size ) == 0 ); + } +# endif + if ( !new_allocation ) [[ unlikely ]] + throw_bad_alloc(); + return new_allocation; + } // crt_aligned_realloc() + + template + [[ using gnu: assume_aligned( alignment ), malloc, returns_nonnull ]] +#ifdef _MSC_VER + __declspec( noalias, restrict ) +#endif + void * crt_realloc( void * const existing_allocation_address, std::size_t const existing_allocation_size, std::size_t const new_size ) + { + if constexpr ( alignment > guaranteed_alignment ) + return crt_aligned_realloc( existing_allocation_address, existing_allocation_size, new_size, alignment ); + else + return crt_realloc( existing_allocation_address, new_size ); + } +#ifdef _MSC_VER + __declspec( noalias ) +#endif + inline + void crt_aligned_free( void * const allocation ) noexcept + { +# ifdef __clang__ + if ( __builtin_constant_p( allocation ) && !allocation ) + return; +# endif +# if defined( _MSC_VER ) + ::_aligned_free( allocation ); +# else + std::free( allocation ); +# endif + } + + template struct capacity { T value; }; + template struct capacity { + constexpr capacity() = default; + constexpr capacity( std::nullptr_t ) noexcept {} + }; +} // namespace detail + +template +struct crt_aligned_allocator +{ + using value_type = T; + using pointer = T *; + using const_pointer = T const *; + using reference = T &; + using const_reference = T const &; + using size_type = sz_t; + using difference_type = std::make_signed_t; + + using allocation_commands = std::uint8_t; + using version = boost::container::dtl::version_type; + + template struct rebind { using other = crt_aligned_allocator; }; + + //!Allocates memory for an array of count elements. + //!Throws bad_alloc if there is no enough memory + //!If Version is 2, this allocated memory can only be deallocated + //!with deallocate() or (for Version == 2) deallocate_many() + [[ nodiscard ]] + [[ using gnu: cold, assume_aligned( alignment ), malloc, returns_nonnull ]] +#ifdef _MSC_VER + __declspec( noalias, restrict ) +#endif + static pointer allocate( size_type const count, [[ maybe_unused ]] void const * const hint = nullptr ) + { + BOOST_ASSUME( count < max_size() ); + auto const byte_size{ count * sizeof( T ) }; + void * new_allocation{ nullptr }; + if constexpr ( alignment > detail::guaranteed_alignment ) + { +# if defined( _MSC_VER ) + new_allocation = ::_aligned_malloc( byte_size, alignment ); +# else + BOOST_VERIFY( posix_memalign( &new_allocation, alignment, byte_size ) == 0 ); +# endif + } + else + { + new_allocation = std::malloc( byte_size ); + } + + if ( !new_allocation ) [[ unlikely ]] + detail::throw_bad_alloc(); + return std::assume_aligned( static_cast( new_allocation ) ); + } + + static auto allocate_at_least( size_type const count ) + { + auto const ptr{ allocate( count ) }; + return std::allocation_result{ ptr, size( ptr ) }; + } + + //!Deallocates previously allocated memory. + //!Never throws + static void deallocate( pointer const ptr, [[ maybe_unused ]] size_type const size ) noexcept + { +# ifdef __clang__ + if ( __builtin_constant_p( ptr ) && !ptr ) + return; +# endif + if constexpr ( alignment > detail::guaranteed_alignment ) + return detail::crt_aligned_free( ptr ); + else + return std::free( ptr ); + } + + [[ nodiscard ]] static pointer resize( pointer const current_address, size_type const current_size, size_type const target_size ) + { + auto const result_address{ do_resize( current_address, current_size, target_size ) }; + if ( !result_address ) [[ unlikely ]] + detail::throw_bad_alloc(); + return result_address; + } + [[ nodiscard ]] static pointer grow_to( pointer const current_address, size_type const current_size, size_type const target_size ) + { + BOOST_ASSUME( target_size >= current_size ); + return resize( current_address, current_size, target_size ); + } + [[ nodiscard ]] static pointer shrink_to( pointer const current_address, size_type const current_size, size_type const target_size ) noexcept + { + BOOST_ASSUME( target_size <= current_size ); + return do_resize( current_address, current_size, target_size ); + } + + //!Returns the maximum number of elements that could be allocated. + //!Never throws + [[ gnu::const ]] static constexpr size_type max_size() noexcept { return std::numeric_limits::max() / sizeof( T ); } + + //!An advanced function that offers in-place expansion shrink to fit and new allocation + //!capabilities. Memory allocated with this function can only be deallocated with deallocate() + //!or deallocate_many(). + // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n2045.html + [[ nodiscard ]] static pointer allocation_command + ( + allocation_commands const command, + [[ maybe_unused ]] + size_type const limit_size, + size_type & prefer_in_recvd_out_size, + pointer & reuse + ) + { + namespace bc = boost::container; + + BOOST_ASSERT_MSG( !( command & bc::zero_memory ), "Unimplemented command" ); + BOOST_ASSERT_MSG( !!( command & bc::shrink_in_place ) != !!( command & ( bc::allocate_new | bc::expand_fwd | bc::expand_bwd ) ), "Conflicting commands" ); + + auto const preferred_size { prefer_in_recvd_out_size }; + auto const preferred_byte_size{ static_cast( preferred_size * sizeof( T ) ) }; + auto const current_size { reuse ? size( reuse ) : 0 }; + bool success{ false }; +# ifdef _MSC_VER // no try_realloc for others so this cannot be safely implemented + // TODO: + // - Linux: switch to mmap+mremap on for non trivial_abi types + // - OSX https://stackoverflow.com/questions/72637850/how-to-use-virtual-memory-implement-realloc-on-mac-osx + if ( reuse && ( command & bc::expand_fwd ) && ( alignment <= detail::guaranteed_alignment ) ) + { + BOOST_ASSUME( preferred_size >= current_size ); + auto const expand_result{ ::_expand( reuse, preferred_byte_size ) }; + if ( expand_result ) + { + BOOST_ASSUME( reuse == expand_result ); + success = true; + } + } + else +# endif + if ( reuse && ( command & ( bc::shrink_in_place | bc::try_shrink_in_place ) ) ) + { + BOOST_ASSUME( preferred_size <= current_size ); + auto const new_address{ std::realloc( reuse, preferred_byte_size ) }; + BOOST_ASSUME( new_address == reuse ); + reuse = new_address; + success = true; + } + else + if ( command & bc::allocate_new ) + { + reuse = allocate( preferred_size ); + success = true; // allocate() throws + } + else + { + std::unreachable(); + } + + if ( success ) [[ likely ]] + { + BOOST_ASSUME( reuse ); + prefer_in_recvd_out_size = size( reuse ); + return reuse; + } + + if ( !( command & bc::nothrow_allocation ) ) + detail::throw_bad_alloc(); + + return nullptr; + } + + //!Returns the maximum number of objects the previously allocated memory + //!pointed by p can hold. + [[ nodiscard, gnu::pure ]] static size_type size( const_pointer const p ) noexcept + { + return static_cast( detail::crt_aligned_alloc_size( p ) / sizeof( T ) ); + } + + //!Allocates just one object. Memory allocated with this function + //!must be deallocated only with deallocate_one(). + //!Throws bad_alloc if there is no enough memory + [[ nodiscard ]] static pointer allocate_one() { return allocate( 1 ); } + //!Deallocates memory previously allocated with allocate_one(). + //!You should never use deallocate_one to deallocate memory allocated + //!with other functions different from allocate_one() or allocate_individual. + //Never throws + static void deallocate_one( pointer const p ) noexcept { return deallocate( p, 1 ); } + +private: + [[ gnu::cold, gnu::assume_aligned( alignment ) ]] + [[ nodiscard ]] static pointer do_resize( pointer const existing_allocation_address, size_type const existing_allocation_size, size_type const new_size ) + { + return std::assume_aligned( static_cast( + detail::crt_realloc( existing_allocation_address, existing_allocation_size * sizeof( T ), new_size * sizeof( T ) ) + )); + } +#if 0 // not implemented/supported/needed yet + using void_multiallocation_chain = boost::container::dtl::basic_multiallocation_chain; + using multiallocation_chain = boost::container::dtl::transform_multiallocation_chain; + + //!Allocates many elements of size == 1. + //!Elements must be individually deallocated with deallocate_one() + void allocate_individual( std::size_t num_elements, multiallocation_chain & chain ) = delete; + + //!Deallocates memory allocated with allocate_one() or allocate_individual(). + //!This function is available only with Version == 2 + void deallocate_individual( multiallocation_chain & chain ) noexcept = delete; + + //!Allocates many elements of size elem_size. + //!Elements must be individually deallocated with deallocate() + void allocate_many( size_type elem_size, std::size_t n_elements, multiallocation_chain & chain ) = delete; + + //!Allocates n_elements elements, each one of size elem_sizes[i] + //!Elements must be individually deallocated with deallocate() + void allocate_many( const size_type * elem_sizes, size_type n_elements, multiallocation_chain & chain ) = delete; + + //!Deallocates several elements allocated by + //!allocate_many(), allocate(), or allocation_command(). + void deallocate_many( multiallocation_chain & chain ) noexcept = delete; +#endif +}; // class crt_aligned_allocator + + +struct crt_vector_options +{ + std::uint8_t alignment { 0 }; // 0 -> default + bool cache_capacity { true }; // if your crt_alloc_size is slow + bool explicit_geometric_growth{ true }; // if your realloc impl is slow (yes MSVC we are again looking at you) +}; // struct crt_vector_options + + +template +requires( is_trivially_moveable ) +class [[ nodiscard, clang::trivial_abi ]] crt_vector + : + public vector_impl, T, sz_t> +{ +public: + static std::uint8_t constexpr alignment{ options.alignment ? options.alignment : std::uint8_t{ alignof( T ) } }; + + using size_type = sz_t; + using value_type = T; + using allocator_type = crt_aligned_allocator; + +private: + using al = allocator_type; + using base = vector_impl, T, sz_t>; + +public: + using base::base; + crt_vector() noexcept : p_array_{ nullptr }, size_{ 0 }, capacity_{ 0 } {} + explicit crt_vector( crt_vector const & other ) + { + auto const data{ storage_init( other.size() ) }; + try { std::uninitialized_copy_n( other.data(), other.size(), data ); } + catch(...) { al::deallocate( data, capacity() ); throw; } + } + crt_vector( crt_vector && other ) noexcept : p_array_{ other.p_array_ }, size_{ other.size_ }, capacity_{ other.capacity_ } { other.mark_freed(); } + + crt_vector & operator=( crt_vector const & other ) { *this = crt_vector( other ); } + crt_vector & operator=( crt_vector && other ) noexcept( std::is_nothrow_move_constructible_v ) + { + std::swap( this->p_array_ , other.p_array_ ); + std::swap( this->size_ , other.size_ ); + std::swap( this->capacity_, other.capacity_ ); + other.free(); + return *this; + } + ~crt_vector() noexcept { free(); } + + [[ nodiscard, gnu::pure ]] size_type size () const noexcept { return size_; } + [[ nodiscard, gnu::pure ]] size_type capacity() const noexcept + { + if constexpr ( options.cache_capacity ) + { + BOOST_ASSUME( capacity_.value >= size_ ); + BOOST_ASSERT( this->empty() || ( capacity_.value <= al::size( p_array_ ) ) ); + return capacity_.value; + } + else + { + return this->empty() ? 0 : al::size( p_array_ ); + } + } + + [[ nodiscard, gnu::pure, gnu::assume_aligned( alignment ) ]] value_type * data() noexcept { return std::assume_aligned( p_array_ ); } + [[ nodiscard, gnu::pure, gnu::assume_aligned( alignment ) ]] value_type const * data() const noexcept { return std::assume_aligned( p_array_ ); } + + void reserve( size_type const new_capacity ) + { + auto const quick_comparison_target{ options.cache_capacity ? capacity() : size() }; + if ( new_capacity > quick_comparison_target ) { + p_array_ = al::grow_to( p_array_, quick_comparison_target, new_capacity ); + update_capacity( new_capacity ); + } + } + +private: friend base; + [[ using gnu: cold, assume_aligned( alignment ), malloc, returns_nonnull, noinline ]] +#ifdef _MSC_VER + __declspec( noalias, restrict ) +#endif + constexpr value_type * storage_init( size_type const initial_size ) + { + p_array_ = al::allocate( initial_size ); + size_ = initial_size; + update_capacity( initial_size ); + return data(); + } + [[ using gnu: assume_aligned( alignment ), returns_nonnull ]] + constexpr value_type * storage_grow_to( size_type const target_size ) + { + auto const current_capacity{ capacity() }; + BOOST_ASSUME( current_capacity >= size_ ); + BOOST_ASSUME( target_size > size_ ); + if ( target_size > current_capacity ) [[ unlikely ]] { + do_grow( target_size, current_capacity ); + } + size_ = target_size; + return data(); + } + + [[ using gnu: assume_aligned( alignment ), returns_nonnull, cold ]] + constexpr value_type * storage_shrink_to( size_type const target_size ) noexcept + { + BOOST_ASSUME( target_size <= size_ ); + p_array_ = al::shrink_to( p_array_, size_, target_size ); + BOOST_ASSUME( p_array_ ); // assuming downsizing never fails + BOOST_ASSUME( is_aligned( p_array_, alignment ) ); // assuming no implementation will actually move a block upon downsizing + storage_shrink_size_to( target_size ); + update_capacity( target_size ); + return data(); + } + constexpr void storage_shrink_size_to( size_type const target_size ) noexcept + { + BOOST_ASSUME( size_ >= target_size ); + size_ = target_size; + } + void storage_dec_size() noexcept { BOOST_ASSUME( size_ >= 1 ); --size_; } + void storage_inc_size() noexcept; // TODO + +private: + [[ gnu::cold, gnu::noinline, clang::preserve_most ]] + void do_grow( size_type const target_size, size_type const cached_current_capacity ) + { + BOOST_ASSUME( cached_current_capacity == capacity() ); + auto const new_capacity + { + options.explicit_geometric_growth + ? std::max( target_size, cached_current_capacity * 3U / 2U ) + : target_size + }; + p_array_ = al::grow_to( p_array_, cached_current_capacity, new_capacity ); + update_capacity( new_capacity ); + } + + void update_capacity( [[ maybe_unused ]] size_type const requested_capacity ) noexcept + { + BOOST_ASSUME( p_array_ ); + if constexpr ( options.cache_capacity ) { +# if defined( _MSC_VER ) + BOOST_ASSERT( al::size( p_array_ ) == requested_capacity ); // see note in crt_alloc_size + capacity_.value = requested_capacity; +# else + capacity_.value = al::size( p_array_ ); + BOOST_ASSUME( capacity_.value >= requested_capacity ); +# endif + } + } + + void free() noexcept + { + std::destroy_n( data(), size() ); + al::deallocate( data(), capacity() ); + mark_freed(); + } + + void mark_freed() noexcept { std::memset( this, 0, sizeof( *this ) ); } + +private: + T * __restrict p_array_; + size_type size_; +#ifdef _MSC_VER + [[ msvc::no_unique_address ]] +#else + [[ no_unique_address ]] +#endif + detail::capacity capacity_; +}; // struct crt_vector + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/is_trivially_moveable.hpp b/include/psi/vm/containers/is_trivially_moveable.hpp new file mode 100644 index 0000000..2dab88f --- /dev/null +++ b/include/psi/vm/containers/is_trivially_moveable.hpp @@ -0,0 +1,68 @@ +//////////////////////////////////////////////////////////////////////////////// +/// +/// Copyright (c) Domagoj Saric. +/// +/// Use, modification and distribution is subject to the +/// Boost Software License, Version 1.0. +/// (See accompanying file LICENSE_1_0.txt or copy at +/// http://www.boost.org/LICENSE_1_0.txt) +/// +/// For more information, see http://www.boost.org +/// +//////////////////////////////////////////////////////////////////////////////// +//------------------------------------------------------------------------------ +#pragma once +#include +//------------------------------------------------------------------------------ +namespace std +{ +#ifdef _LIBCPP_VERSION +inline namespace __1 { +#endif + template class vector; + template class basic_string; + template class unique_ptr; + template class function; +#ifdef _LIBCPP_VERSION +} // namespace __1 +#endif +} // namespace std +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p1144r12.html std::is_trivially_relocatable +// https://quuxplusone.github.io/blog/2019/02/20/p1144-what-types-are-relocatable +// https://github.com/Quuxplusone/libcxx/blob/trivially-relocatable/test/libcxx/type_traits/is_trivially_relocatable.pass.cpp +// https://brevzin.github.io/c++/2024/10/21/trivial-relocation +// https://github.com/abseil/abseil-cpp/pull/1625 +// https://reviews.llvm.org/D114732 +// https://github.com/llvm/llvm-project/pull/88857 +// +// allowed to be user-specialized for custom types +template +bool constexpr is_trivially_moveable +{ +#ifdef __clang__ + __is_trivially_relocatable( T ) || +#endif +#ifdef __cpp_lib_trivially_relocatable + std::is_trivially_relocatable || +#endif + std::is_trivially_move_constructible_v || + std::is_trivially_copyable_v +}; // is_trivially_moveable + +#if !defined( _LIBCPP_DEBUG ) +template bool constexpr is_trivially_moveable>{ is_trivially_moveable }; +template bool constexpr is_trivially_moveable>{ is_trivially_moveable }; +template bool constexpr is_trivially_moveable>{ is_trivially_moveable }; +#endif +#if defined( __GLIBCXX__ ) +template bool constexpr is_trivially_moveable>{ true }; +#endif + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/static_vector.hpp b/include/psi/vm/containers/static_vector.hpp new file mode 100644 index 0000000..8f1a54f --- /dev/null +++ b/include/psi/vm/containers/static_vector.hpp @@ -0,0 +1,172 @@ +//////////////////////////////////////////////////////////////////////////////// +/// Yet another take on prior art a la boost::container::static_vector and +/// std::inplace_vector with emphasis on improved debuggability w/o custom type +/// visualizers (i.e. seeing the contained values rather than random bytes), +/// maximum efficiency (avoiding conditionals and dynamic memcpy calls for small +/// vectors - utilizing large SIMD registers for inlined copies with a handful +/// of instructions) and configurability (e.g. overflow handler), in addition +/// to extentions provided by vector_impl. +//////////////////////////////////////////////////////////////////////////////// +/// +/// Copyright (c) Domagoj Saric. +/// +/// Use, modification and distribution is subject to the +/// Boost Software License, Version 1.0. +/// (See accompanying file LICENSE_1_0.txt or copy at +/// http://www.boost.org/LICENSE_1_0.txt) +/// +/// For more information, see http://www.boost.org +/// +//////////////////////////////////////////////////////////////////////////////// +//------------------------------------------------------------------------------ +#pragma once + +#include + +#include +#include + +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +template +union noninitialized_array +{ + constexpr noninitialized_array() noexcept {} + constexpr ~noninitialized_array() noexcept {} + + T data[ size ]; +}; // noninitialized_array + +struct assert_on_overflow { + [[ noreturn ]] static void operator()() noexcept { + BOOST_ASSERT_MSG( false, "Static vector overflow!" ); + std::unreachable(); + } +}; // assert_on_overflow +struct throw_on_overflow { + [[ noreturn ]] static void operator()() { detail::throw_out_of_range(); } +}; // throw_on_overflow + +template +class [[ clang::trivial_abi ]] static_vector + : + public vector_impl, T, typename boost::uint_value_t::least> +{ +public: + using size_type = typename boost::uint_value_t::least; + using value_type = T; + + static size_type constexpr static_capacity{ maximum_size }; + +private: + using base = vector_impl, T, typename boost::uint_value_t::least>; + + // https://github.com/llvm/llvm-project/issues/54535 + // https://github.com/llvm/llvm-project/issues/42585 + // https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/X86/X86Subtarget.h#L78 + // https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/ARM/ARMSubtarget.h#L216 + static auto constexpr unconditional_fixed_memcopy_size_limit + { +# ifdef __AVX512F__ + 256 +# else + 128 +# endif + }; + struct this_pod { size_type _0; noninitialized_array _1; }; + static bool constexpr fixed_sized_copy{ std::is_trivially_copy_constructible_v && ( sizeof( this_pod ) <= unconditional_fixed_memcopy_size_limit ) }; + +public: + using base::base; + constexpr static_vector() noexcept : size_{ 0 } {} + constexpr explicit static_vector( static_vector const & other ) noexcept( std::is_nothrow_copy_constructible_v ) + { + if constexpr ( fixed_sized_copy ) { + fixed_copy( other ); + } else { + std::uninitialized_copy_n( other.data(), other.size(), this->data() ); + this->size_ = other.size(); + } + } + constexpr static_vector( static_vector && other ) noexcept( std::is_nothrow_move_constructible_v ) + { + if constexpr ( fixed_sized_copy ) { + fixed_copy( other ); + } else { + std::uninitialized_move_n( other.data(), other.size(), this->data() ); + this->size_ = other.size(); + } + other.size_ = 0; + } + constexpr static_vector & operator=( static_vector const & other ) noexcept( std::is_nothrow_copy_constructible_v ) + { + if constexpr ( fixed_sized_copy ) { + fixed_copy( other ); + return *this; + } else { + return static_cast( base::operator=( other ) ); + } + } + constexpr static_vector & operator=( static_vector && other ) noexcept( std::is_nothrow_move_constructible_v ) + { + if constexpr ( fixed_sized_copy ) { + fixed_copy( other ); + return *this; + } else { + return static_cast( base::operator=( std::move( other ) ) ); + } + } + + [[ nodiscard, gnu::pure ]] constexpr size_type size () const noexcept { BOOST_ASSUME( size_ <= maximum_size ); return size_; } + [[ nodiscard, gnu::const ]] static constexpr size_type capacity() noexcept { return maximum_size; } + + [[ nodiscard, gnu::const ]] constexpr value_type * data() noexcept { return array_.data; } + [[ nodiscard, gnu::const ]] constexpr value_type const * data() const noexcept { return array_.data; } + + void reserve( size_type const new_capacity ) noexcept { BOOST_ASSUME( new_capacity <= maximum_size ); } + +private: friend base; + constexpr value_type * storage_init( size_type const initial_size ) noexcept( noexcept( overflow_handler() ) ) { return storage_grow_to( initial_size ); } + constexpr value_type * storage_grow_to( size_type const target_size ) noexcept( noexcept( overflow_handler() ) ) + { + static_assert( sizeof( *this ) == sizeof( this_pod ) ); + if ( target_size > maximum_size ) [[ unlikely ]] { + overflow_handler(); + } + size_ = target_size; + return data(); + } + + constexpr value_type * storage_shrink_to( size_type const target_size ) noexcept + { + storage_shrink_size_to( target_size ); + return data(); + } + constexpr void storage_shrink_size_to( size_type const target_size ) noexcept + { + BOOST_ASSUME( size_ >= target_size ); + size_ = target_size; + } + void storage_dec_size() noexcept { BOOST_ASSUME( size_ >= 1 ); --size_; } + void storage_inc_size() noexcept; // TODO + +private: + void fixed_copy( static_vector const & __restrict source ) noexcept + requires( fixed_sized_copy ) + { + std::memcpy( this, &source, sizeof( *this ) ); + BOOST_ASSUME( this->size_ == source.size() ); + } + +private: + size_type size_; + noninitialized_array array_; +}; // struct static_vector + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/vector.hpp b/include/psi/vm/containers/vector.hpp index c1e66f2..fe4f02a 100644 --- a/include/psi/vm/containers/vector.hpp +++ b/include/psi/vm/containers/vector.hpp @@ -13,23 +13,13 @@ //------------------------------------------------------------------------------ #pragma once -#include +#include #include #include #include #include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include //------------------------------------------------------------------------------ namespace psi::vm { @@ -37,36 +27,52 @@ namespace psi::vm namespace detail { - [[ noreturn, gnu::cold ]] void throw_out_of_range(); - template struct size { T value; }; template struct size {}; } // namespace detail -class contiguous_container_storage_base +class [[ clang::trivial_abi ]] contiguous_container_storage_base { public: + // TODO shallow/COW copy construction, + OSX vm_copy contiguous_container_storage_base( contiguous_container_storage_base && ) = default; contiguous_container_storage_base & operator=( contiguous_container_storage_base && ) = default; - auto data() noexcept { BOOST_ASSERT_MSG( mapping_, "Paging file not attached" ); return std::assume_aligned( view_.data() ); } - auto data() const noexcept { return const_cast( *this ).data(); } + [[ nodiscard, gnu::pure, gnu::assume_aligned( reserve_granularity ) ]] auto * data() noexcept { BOOST_ASSERT_MSG( mapping_, "Paging file not attached" ); return std::assume_aligned( view_.data() ); } + [[ nodiscard, gnu::pure, gnu::assume_aligned( reserve_granularity ) ]] auto * data() const noexcept { return const_cast( *this ).data(); } + + //! Effects: Tries to deallocate the excess of memory created + //! with previous allocations. The size of the vector is unchanged + //! + //! Throws: nothing. + //! + //! Complexity: Constant. + void shrink_to_fit() noexcept; void unmap() noexcept { view_.unmap(); } void close() noexcept; - [[ gnu::pure ]] auto storage_size() const noexcept { return get_size( mapping_ ); } - [[ gnu::pure ]] auto mapped_size() const noexcept { return view_.size(); } + [[ nodiscard, gnu::pure ]] auto storage_size() const noexcept { return get_size( mapping_ ); } + [[ nodiscard, gnu::pure ]] auto mapped_size() const noexcept { return view_.size(); } + + void flush_async () const noexcept { flush_async ( 0, mapped_size() ); } + void flush_blocking() const noexcept { flush_blocking( 0, mapped_size() ); } void flush_async ( std::size_t beginning, std::size_t size ) const noexcept; void flush_blocking( std::size_t beginning, std::size_t size ) const noexcept; - bool file_backed() const noexcept { return mapping_.is_file_based(); } + [[ nodiscard, gnu::pure ]] bool file_backed() const noexcept { return mapping_.is_file_based(); } + + [[ nodiscard, gnu::pure ]] bool has_attached_storage() const noexcept { return static_cast( mapping_ ); } - explicit operator bool() const noexcept { return static_cast( mapping_ ); } + explicit operator bool() const noexcept { return has_attached_storage(); } + + void swap( contiguous_container_storage_base & other ) noexcept { std::swap( *this, other ); } protected: + static bool constexpr storage_zero_initialized{ true }; + constexpr contiguous_container_storage_base() = default; err::fallible_result @@ -75,25 +81,24 @@ class contiguous_container_storage_base return map_file( create_file( file_name, create_rw_file_flags( policy ) ), header_size ); } + // template (char type) independent portion of map_file err::fallible_result - map_memory( std::size_t const size ) noexcept { return map( {}, size ); } - - - void expand( std::size_t target_size ); + map_file( file_handle && file, std::size_t header_size ) noexcept; - void expand_view( std::size_t target_size ); + err::fallible_result + map_memory( std::size_t const size ) noexcept { return map( {}, size ); } - void shrink( std::size_t target_size ) noexcept; + void reserve( std::size_t new_capacity ); + void * resize ( std::size_t target_size ); - void shrink_to_fit() noexcept; + void * shrink_to( std::size_t target_size ) noexcept( mapping::views_downsizeable ); + void * grow_to( std::size_t target_size ); - void resize( std::size_t target_size ); + void * expand_view( std::size_t target_size ); - void reserve( std::size_t new_capacity ); + void shrink_mapped_size_to( std::size_t target_size ) noexcept( mapping::views_downsizeable ); - // template (char type) independent portion of map_file - err::fallible_result - map_file( file_handle && file, std::size_t header_size ) noexcept; + void free() noexcept; private: err::fallible_result @@ -105,12 +110,13 @@ class contiguous_container_storage_base }; // contiguous_container_storage_base template -class contiguous_container_storage +class [[ clang::trivial_abi ]] contiguous_container_storage : public contiguous_container_storage_base, private detail::size - // checkout a revision prior to March the 21st 2024 for a version that used statically sized header sizes - // this approach is more versatile while the overhead should be less than negligible + // Checkout a revision prior to March the 21st 2024 for a version that used + // statically sized header sizes. The current approach is more versatile + // while the overhead should be less than negligible. { public: using size_type = sz_t; @@ -170,45 +176,53 @@ class contiguous_container_storage auto data() noexcept { return contiguous_container_storage_base::data() + header_size(); } auto data() const noexcept { return contiguous_container_storage_base::data() + header_size(); } - [[ gnu::pure, nodiscard ]] auto storage_size() const noexcept { return static_cast( contiguous_container_storage_base::storage_size() ); } - [[ gnu::pure, nodiscard ]] auto mapped_size() const noexcept { return static_cast( contiguous_container_storage_base:: mapped_size() ); } + //! Effects: Returns true if the vector contains no elements. + //! Throws: Nothing. + //! Complexity: Constant. + [[ nodiscard, gnu::pure ]] bool empty() const noexcept { return !has_attached_storage() || !size(); } + + [[ nodiscard, gnu::pure ]] auto storage_size() const noexcept { return static_cast( contiguous_container_storage_base::storage_size() ); } + [[ nodiscard, gnu::pure ]] auto mapped_size() const noexcept { return static_cast( contiguous_container_storage_base:: mapped_size() ); } - [[ gnu::pure, nodiscard ]] auto size () const noexcept { if constexpr ( headerless ) return mapped_size(); else return stored_size(); } - [[ gnu::pure, nodiscard ]] auto capacity() const noexcept { if constexpr ( headerless ) return storage_size(); else return mapped_size() - header_size(); } + [[ nodiscard, gnu::pure ]] auto size () const noexcept { if constexpr ( headerless ) return mapped_size(); else return stored_size(); } + [[ nodiscard, gnu::pure ]] auto capacity() const noexcept { if constexpr ( headerless ) return storage_size(); else return mapped_size() - header_size(); } - void expand( size_type const target_size ) +protected: + void * grow_to( size_type const target_size ) { BOOST_ASSUME( target_size >= size() ); if constexpr ( headerless ) { if ( target_size > size() ) - contiguous_container_storage_base::expand( target_size ); + contiguous_container_storage_base::grow_to( target_size ); } else { if ( target_size > capacity() ) - contiguous_container_storage_base::expand( target_size + header_size() ); + contiguous_container_storage_base::grow_to( target_size + header_size() ); stored_size() = target_size; } + return data(); } - void shrink( size_type const target_size ) noexcept + void * shrink_to( size_type const target_size ) noexcept { - contiguous_container_storage_base::shrink( target_size + header_size() ); + auto const data_ptr{ contiguous_container_storage_base::shrink_to( target_size + header_size() ) }; if constexpr ( !headerless ) stored_size() = target_size; + return data_ptr; } void resize( size_type const target_size ) { if ( target_size > size() ) - { + { // partial replication of grow_to logic to avoid a double target_size > size() check if constexpr ( headerless ) - expand( target_size ); + grow_to( target_size ); else { if ( target_size > capacity() ) - expand( target_size ); + grow_to( target_size ); else stored_size() = target_size; } @@ -216,10 +230,12 @@ class contiguous_container_storage else { // or skip this like std::vector and rely on an explicit shrink_to_fit() call? - shrink( target_size ); + shrink_to( target_size ); } if constexpr ( !headerless ) - { BOOST_ASSUME( stored_size() == target_size ); } + { + BOOST_ASSUME( stored_size() == target_size ); + } } void reserve( size_type const new_capacity ) @@ -228,16 +244,15 @@ class contiguous_container_storage contiguous_container_storage_base::reserve( new_capacity ); else if ( new_capacity > capacity() ) - contiguous_container_storage_base::expand( new_capacity + header_size() ); + contiguous_container_storage_base::grow_to( new_capacity + header_size() ); } void shrink_to_fit() noexcept { if constexpr ( headerless ) contiguous_container_storage_base::shrink_to_fit(); - else { - contiguous_container_storage_base::shrink( stored_size() + header_size() ); - } + else + contiguous_container_storage_base::shrink_to( stored_size() + header_size() ); } bool has_extra_capacity() const noexcept @@ -246,39 +261,122 @@ class contiguous_container_storage return size() != capacity(); } - void allocate_available_capacity( size_type const sz ) noexcept + void grow_into_available_capacity_by( size_type const sz_delta ) noexcept { - BOOST_ASSERT_MSG( sz <= ( capacity() - size() ), "Out of preallocated space" ); + BOOST_ASSERT_MSG( sz_delta <= ( capacity() - size() ), "Out of preallocated space" ); if constexpr ( headerless ) - contiguous_container_storage_base::expand_view( mapped_size() + sz ); + contiguous_container_storage_base::expand_view( mapped_size() + sz_delta ); else - stored_size() += sz; + stored_size() += sz_delta; } -private: - [[ gnu::pure, nodiscard ]] size_type & stored_size() noexcept requires( !headerless ) + void shrink_size_to( sz_t const new_size ) noexcept + { + BOOST_ASSUME( new_size <= capacity() ); + if constexpr ( headerless ) + contiguous_container_storage_base::shrink_mapped_size_to( new_size ); + else + stored_size() = new_size; + } + +protected: + void free() noexcept + { + if constexpr ( headerless ) + contiguous_container_storage_base::free(); + else + shrink_to( 0 ); + } + + [[ nodiscard, gnu::pure ]] size_type & stored_size() noexcept requires( !headerless ) { auto const p_size{ contiguous_container_storage_base::data() + header_size() - size_size }; BOOST_ASSERT( reinterpret_cast( p_size ) % alignof( size_type ) == 0 ); return *reinterpret_cast( p_size ); } - [[ gnu::pure, nodiscard ]] size_type stored_size() const noexcept requires( !headerless ) + [[ nodiscard, gnu::pure ]] size_type stored_size() const noexcept requires( !headerless ) { return const_cast( *this ).stored_size(); } }; // contiguous_container_storage -template < typename T > -bool constexpr is_trivially_moveable +template +class [[ clang::trivial_abi ]] typed_contiguous_container_storage + : + public contiguous_container_storage, + public vector_impl, T, sz_t> { -#ifdef __clang__ - __is_trivially_relocatable( T ) || -#endif - std::is_trivially_move_constructible_v -}; +private: + using base = contiguous_container_storage; + using vec_impl = vector_impl, T, sz_t>; + +public: + using value_type = T; + + using base::base; + + using base::empty; + using vec_impl::grow_to; + using vec_impl::shrink_to; + using vec_impl::resize; + + [[ nodiscard, gnu::pure ]] T * data() noexcept { return to_t_ptr( base::data() ); } + //using vec_impl::data; + [[ nodiscard, gnu::pure ]] T const * data() const noexcept { return const_cast( *this ).data(); } + + //! Effects: Returns the number of the elements contained in the vector. + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + [[ nodiscard, gnu::pure ]] sz_t size() const noexcept { return to_t_sz( base::size() ); } + + //! Effects: Number of elements for which memory has been allocated. + //! capacity() is always greater than or equal to size(). + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + [[ nodiscard, gnu::pure ]] sz_t capacity() const noexcept{ return to_t_sz( base::capacity() ); } + + void reserve( sz_t const new_capacity ) { base::reserve( to_byte_sz( new_capacity ) ); } + + decltype( auto ) user_header_data() noexcept { return base::header_storage(); } + + // helper getter for generic code that wishes to do basic manipulation on + // vm::vectors w/o being templated (contiguous_container_storage_base does not + // publicize functionality that could be used to make it out of sync with the + // corresponding vm::vector) + contiguous_container_storage_base & storage_base() noexcept { return *this; } + +private: friend vec_impl; + //! Effects: If n is less than or equal to capacity(), this call has no + //! effect. Otherwise, it is a request for allocation of additional memory. + //! If the request is successful, then capacity() is greater than or equal to + //! n; otherwise, capacity() is unchanged. In either case, size() is unchanged. + //! + //! Throws: If memory allocation allocation throws or T's copy/move constructor throws. + T * storage_shrink_to( sz_t const target_size ) noexcept { return static_cast( base::shrink_to( to_byte_sz( target_size ) ) ); } + T * storage_grow_to ( sz_t const target_size ) { return static_cast( base::grow_to ( to_byte_sz( target_size ) ) ); } + + void storage_shrink_size_to( sz_t const new_size ) noexcept { base::shrink_size_to( to_byte_sz( new_size ) ); } + void storage_dec_size() noexcept { storage_shrink_size_to( size() - 1 ); } + void storage_inc_size() noexcept; // TODO + +protected: + PSI_WARNING_DISABLE_PUSH() + PSI_WARNING_GCC_OR_CLANG_DISABLE( -Wsign-conversion ) + static T * to_t_ptr ( mapped_view::value_type * const ptr ) noexcept { return reinterpret_cast( ptr ); } + static sz_t to_t_sz ( auto const byte_sz ) noexcept { BOOST_ASSUME( byte_sz % sizeof( T ) == 0 ); return static_cast( byte_sz / sizeof( T ) ); } + static sz_t to_byte_sz( auto const sz ) noexcept + { + auto const rez{ sz * sizeof( T ) }; + BOOST_ASSERT( rez <= std::numeric_limits::max() ); + return static_cast( rez ); + } + PSI_WARNING_DISABLE_POP() +}; // class typed_contiguous_container_storage -struct default_init_t{}; inline constexpr default_init_t default_init; -struct value_init_t {}; inline constexpr value_init_t value_init ; struct header_info { @@ -322,40 +420,21 @@ struct header_info template requires is_trivially_moveable -class vector +class [[ clang::trivial_abi ]] vector + : + public typed_contiguous_container_storage { private: - using storage_t = contiguous_container_storage; - storage_t storage_; + using storage_t = typed_contiguous_container_storage; + //using impl = vector_impl; + using impl = storage_t; public: static constexpr auto headerless{ headerless_param }; - using span_t = std::span; - using const_span_t = std::span; - - using value_type = T; - using pointer = T *; - using const_pointer = T const *; - using reference = T &; - using const_reference = T const &; - using param_const_ref = std::conditional_t< std::is_trivial_v && ( sizeof( T ) <= 2 * sizeof( void * ) ), T, const_reference >; - using size_type = sz_t; - using difference_type = std::make_signed_t; - using iterator = typename span_t:: iterator; - using reverse_iterator = typename span_t:: reverse_iterator; -#if defined( _LIBCPP_VERSION ) // no span::const_iterators yet (as of v17) - using const_iterator = typename const_span_t::iterator; - using const_reverse_iterator = typename const_span_t::reverse_iterator; -#else - using const_iterator = typename span_t::const_iterator; - using const_reverse_iterator = typename span_t::const_reverse_iterator; -#endif - // not really a standard allocator: providing the alias simply to have boost::container::flat* compileable with this container. - using allocator_type = std::allocator; + using allocator_type = std::allocator; -public: vector() noexcept requires( headerless ) {} // Allowing for a header to store/persist the 'actual' size of the container can be generalized // to storing arbitrary (types of) headers. This however makes this class template no longer @@ -365,863 +444,48 @@ class vector // Awaiting a better name for the idiom, considering its usefulness, the library exposes this // ability/functionality publicly - as a runtime parameter (instead of a Header template type // parameter) - the relative runtime cost should be near non-existent vs all the standard - // benefits (having concrete base clasess, less template instantiations and codegen copies...). + // benefits (having concrete base classes, less template instantiations and codegen copies...). explicit vector( header_info const hdr_info = {} ) noexcept requires( !headerless ) // TODO: use slack space (if any) in the object placed (by user code) in the header space // to store the size (to avoid wasting even more slack space due to alignment padding // after appending the size ('member'). : - storage_ + storage_t ( hdr_info - . add_header() - .template with_alignment_for() + . add_header() + .template with_alignment_for() . final_header_size() ) {} - vector( vector const & ) = delete; - vector( vector && ) = default; + vector( vector const & ) = delete; + vector( vector && ) = default; ~vector() noexcept = default; - vector & operator=( vector && ) = default; - vector & operator=( vector const & x ) - { - BOOST_ASSUME( &x != this ); // not going to support self assignment - assign( x ); - return *this; - } - - vector & operator=( std::initializer_list const data ) { this->assign( data ); return *this; } - - //! Effects: Assigns the the range [first, last) to *this. - //! - //! Throws: If memory allocation throws or T's copy/move constructor/assignment or - //! T's constructor/assignment from dereferencing InpIt throws. - //! - //! Complexity: Linear to n. - template < std::input_iterator It> - void assign( It first, It const last ) - { - //Overwrite all elements we can from [first, last) - auto cur { this->begin() }; - auto const end_it{ this->end () }; - for ( ; first != last && cur != end_it; ++cur, ++first ) - *cur = *first; - - if ( first == last ) - { - auto const target_size{ static_cast( cur - begin() ) }; - //There are no more elements in the sequence, erase remaining - while ( cur != end_it ) - (*cur++).~value_type(); - shrink_storage_to( target_size ); - } - else - { - //There are more elements in the range, insert the remaining ones - this->append_range( std::span{ first, last } ); - } - } - - void assign( std::initializer_list const data ) { this->assign( data.begin(), data.end() ); } - - //! Effects: Assigns the n copies of val to *this. - //! - //! Throws: If memory allocation throws or - //! T's copy/move constructor/assignment throws. - //! - //! Complexity: Linear to n. - void assign( size_type const n, param_const_ref val ) = /*TODO*/delete; - - //! Effects: Returns a copy of the internal allocator. - //! - //! Throws: If allocator's copy constructor throws. - //! - //! Complexity: Constant. - auto const & get_stored_allocator() const noexcept { return storage_; } - - ////////////////////////////////////////////// - // - // iterators - // - ////////////////////////////////////////////// - - //! Effects: Returns an iterator to the first element contained in the vector. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] auto begin() noexcept { return span().begin(); } - [[ nodiscard ]] auto begin() const noexcept { return cbegin(); } - [[ nodiscard ]] auto cbegin() const noexcept -#if defined( _MSC_VER ) - { return const_cast( *this ).span().cbegin(); } // wrkrnd for span and span iterators not being compatible w/ MS STL -#else - { return span().begin(); } -#endif - - //! Effects: Returns an iterator to the end of the vector. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] auto end() noexcept { return span().end(); } - [[ nodiscard ]] auto end() const noexcept { return cend(); } - [[ nodiscard ]] auto cend() const noexcept -#if defined( _MSC_VER ) - { return const_cast( *this ).span().cend(); } -#else - { return span().end(); } -#endif - - //! Effects: Returns a reverse_iterator pointing to the beginning - //! of the reversed vector. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] auto rbegin() noexcept { return span().rbegin(); } - [[ nodiscard ]] auto rbegin() const noexcept { return span().rbegin(); } - [[ nodiscard ]] auto crbegin() const noexcept { return rbegin(); } - - //! Effects: Returns a reverse_iterator pointing to the end - //! of the reversed vector. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] auto rend() noexcept { return span().rend(); } - [[ nodiscard ]] auto rend() const noexcept { return static_cast( const_cast( *this ).span() ).rend(); } - [[ nodiscard ]] auto crend() const noexcept { return rend(); } - - ////////////////////////////////////////////// - // - // capacity - // - ////////////////////////////////////////////// - - //! Effects: Returns true if the vector contains no elements. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard, gnu::pure ]] bool empty() const noexcept { return !storage_ || span().empty(); } - - //! Effects: Returns the number of the elements contained in the vector. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard, gnu::pure ]] size_type size() const noexcept { return to_t_sz( storage_.size() ); } - - //! Effects: Returns the largest possible size of the vector. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] static constexpr size_type max_size() noexcept { return static_cast( std::numeric_limits< size_type >::max() / sizeof( value_type ) ); } - - //! Effects: Inserts or erases elements at the end such that - //! the size becomes n. New elements can be default or value initialized. - //! - //! Throws: If memory allocation throws, or T's copy/move or value initialization throws. - //! - //! Complexity: Linear to the difference between size() and new_size. - void resize( size_type const new_size, default_init_t ) requires( std::is_trivial_v ) - { - storage_.resize( to_byte_sz( new_size ) ); - } - void resize( size_type const new_size, value_init_t ) - { - if ( new_size > size() ) grow_to( new_size, value_init ); - else shrink_to( new_size ); - } - - //! Effects: Inserts or erases elements at the end such that - //! the size becomes n. New elements are copy constructed from x. - //! - //! Throws: If memory allocation throws, or T's copy/move constructor throws. - //! - //! Complexity: Linear to the difference between size() and new_size. - void resize( size_type const new_size, param_const_ref x ) - { - auto const current_size{ size() }; - BOOST_ASSUME( new_size >= current_size ); - storage_.resize( to_byte_sz( new_size ) ); - auto uninitialized_span{ span().subspan( current_size ) }; - std::uninitialized_fill( uninitialized_span.begin(), uninitialized_span.end(), x ); - } - - //! Effects: Number of elements for which memory has been allocated. - //! capacity() is always greater than or equal to size(). - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] size_type capacity() const noexcept{ return to_t_sz( storage_.capacity() ); } - - //! Effects: If n is less than or equal to capacity(), this call has no - //! effect. Otherwise, it is a request for allocation of additional memory. - //! If the request is successful, then capacity() is greater than or equal to - //! n; otherwise, capacity() is unchanged. In either case, size() is unchanged. - //! - //! Throws: If memory allocation allocation throws or T's copy/move constructor throws. - void reserve( size_type const new_capacity ) { storage_.reserve( to_byte_sz( new_capacity ) ); } - - //! Effects: Tries to deallocate the excess of memory created - //! with previous allocations. The size of the vector is unchanged - //! - //! Throws: If memory allocation throws, or T's copy/move constructor throws. - //! - //! Complexity: Linear to size(). - void shrink_to_fit() { storage_.shrink_to_fit(); } - - ////////////////////////////////////////////// - // - // element access - // - ////////////////////////////////////////////// - - //! Requires: !empty() - //! - //! Effects: Returns a reference to the first - //! element of the container. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] reference front() noexcept { return span().front(); } - - //! Requires: !empty() - //! - //! Effects: Returns a const reference to the first - //! element of the container. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] const_reference front() const noexcept { return span().front(); } - - //! Requires: !empty() - //! - //! Effects: Returns a reference to the last - //! element of the container. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] reference back() noexcept { return span().back(); } - - //! Requires: !empty() - //! - //! Effects: Returns a const reference to the last - //! element of the container. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] const_reference back() const noexcept { return span().back(); } - - //! Requires: size() > n. - //! - //! Effects: Returns a reference to the nth element - //! from the beginning of the container. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] reference operator[]( size_type const n ) noexcept { return span()[ n ]; } - - //! Requires: size() > n. - //! - //! Effects: Returns a const reference to the nth element - //! from the beginning of the container. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] const_reference operator[]( size_type const n ) const noexcept { return span()[ n ]; } - - //! Requires: size() >= n. - //! - //! Effects: Returns an iterator to the nth element - //! from the beginning of the container. Returns end() - //! if n == size(). - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - //! - //! Note: Non-standard extension - [[ nodiscard ]] iterator nth( size_type const n ) noexcept - { - BOOST_ASSERT( n <= size() ); - return begin() + static_cast( n ); - } - - //! Requires: size() >= n. - //! - //! Effects: Returns a const_iterator to the nth element - //! from the beginning of the container. Returns end() - //! if n == size(). - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - //! - //! Note: Non-standard extension - [[ nodiscard ]] const_iterator nth( size_type const n ) const noexcept - { - BOOST_ASSERT( n <= size() ); - return begin() + static_cast( n ); - } - - //! Requires: begin() <= p <= end(). - //! - //! Effects: Returns the index of the element pointed by p - //! and size() if p == end(). - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - //! - //! Note: Non-standard extension - [[ nodiscard ]] size_type index_of( iterator const p ) noexcept - { - verify_iterator( p ); - return static_cast( p - begin() ); - } - - //! Requires: begin() <= p <= end(). - //! - //! Effects: Returns the index of the element pointed by p - //! and size() if p == end(). - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - //! - //! Note: Non-standard extension - [[ nodiscard ]] size_type index_of( const_iterator const p ) const noexcept - { - verify_iterator( p ); - return static_cast( p - begin() ); - } - - //! Requires: size() > n. - //! - //! Effects: Returns a reference to the nth element - //! from the beginning of the container. - //! - //! Throws: range_error if n >= size() - //! - //! Complexity: Constant. - [[ nodiscard ]] reference at( size_type const n ) - { - if constexpr ( requires{ span().at( n ); } ) // cpp23 - return span().at( n ); - else - { - if ( n >= size() ) - detail::throw_out_of_range(); - return (*this)[ n ]; - } - } - - //! Requires: size() > n. - //! - //! Effects: Returns a const reference to the nth element - //! from the beginning of the container. - //! - //! Throws: range_error if n >= size() - //! - //! Complexity: Constant. - [[ nodiscard ]] const_reference at( size_type const n ) const - { - if constexpr ( requires{ span().at( n ); } ) // cpp23 - return span().at( n ); - else - { - if ( n >= size() ) - detail::throw_out_of_range(); - return (*this)[ n ]; - } - } - - ////////////////////////////////////////////// - // - // data access - // - ////////////////////////////////////////////// - - //! Returns: A pointer such that [data(),data() + size()) is a valid range. - //! For a non-empty vector, data() == &front(). - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - [[ nodiscard ]] T * data() noexcept { return to_t_ptr( storage_.data() ); } - [[ nodiscard ]] T const * data() const noexcept { return const_cast( *this ).data(); } - - [[ nodiscard ]] span_t span() noexcept { return { data(), size() }; } - [[ nodiscard ]] const_span_t span() const noexcept { return { data(), size() }; } - - ////////////////////////////////////////////// - // - // modifiers - // - ////////////////////////////////////////////// - - //! Effects: Inserts an object of type T constructed with - //! std::forward(args)... in the end of the vector. - //! - //! Returns: A reference to the created object. - //! - //! Throws: If memory allocation throws or the in-place constructor throws or - //! T's copy/move constructor throws. - //! - //! Complexity: Amortized constant time. - template - reference emplace_back( Args &&...args ) - { - storage_.expand( to_byte_sz( size() + 1 ) ); - auto & placeholder{ back() }; - if constexpr ( sizeof...( args ) ) - return *std::construct_at( &placeholder, std::forward( args )... ); - else - return *new ( &placeholder ) T; // default init - } - - //! Effects: Inserts an object of type T constructed with - //! std::forward(args)... in the end of the vector. - //! - //! Throws: If the in-place constructor throws. - //! - //! Complexity: Constant time. - //! - //! Note: Non-standard extension. - template - bool stable_emplace_back(Args &&...args) = /*TODO*/delete; - - //! Requires: position must be a valid iterator of *this. - //! - //! Effects: Inserts an object of type T constructed with - //! std::forward(args)... before position - //! - //! Throws: If memory allocation throws or the in-place constructor throws or - //! T's copy/move constructor/assignment throws. - //! - //! Complexity: If position is end(), amortized constant time - //! Linear time otherwise. - template - iterator emplace( const_iterator const position, Args && ...args ) - { - auto const iter{ make_space_for_insert( position, 1 ) }; - std::construct_at( &*iter, std::forward< Args >( args )... ); - return iter; - } - - //! Effects: Inserts a copy of x at the end of the vector. - //! - //! Throws: If memory allocation throws or - //! T's copy/move constructor throws. - //! - //! Complexity: Amortized constant time. - void push_back( param_const_ref x ) { emplace_back( x ); } - - //! Effects: Constructs a new element in the end of the vector - //! and moves the resources of x to this new element. - //! - //! Throws: If memory allocation throws or - //! T's copy/move constructor throws. - //! - //! Complexity: Amortized constant time. - void push_back( T && x ) requires( !std::is_trivial_v ) { emplace_back( std::move( x ) ); } - - //! Requires: position must be a valid iterator of *this. - //! - //! Effects: Insert a copy of x before position. - //! - //! Throws: If memory allocation throws or T's copy/move constructor/assignment throws. - //! - //! Complexity: If position is end(), amortized constant time - //! Linear time otherwise. - iterator insert( const_iterator const position, param_const_ref x ) { return emplace( position, x ); } - - //! Requires: position must be a valid iterator of *this. - //! - //! Effects: Insert a new element before position with x's resources. - //! - //! Throws: If memory allocation throws. - //! - //! Complexity: If position is end(), amortized constant time - //! Linear time otherwise. - iterator insert( const_iterator const position, T && x ) requires( !std::is_trivial_v ) { return emplace( position, std::move( x ) ); } - - //! Requires: position must be a valid iterator of *this. - //! - //! Effects: Insert n copies of x before pos. - //! - //! Returns: an iterator to the first inserted element or p if n is 0. - //! - //! Throws: If memory allocation throws or T's copy/move constructor throws. - //! - //! Complexity: Linear to n. - iterator insert( const_iterator const position, size_type const n, param_const_ref x ) - { - auto const iter{ make_space_for_insert( position, n ) }; - std::fill_n( iter, n, x ); - return iter; - } - - //! Requires: position must be a valid iterator of *this. - //! - //! Effects: Insert a copy of the [first, last) range before pos. - //! - //! Returns: an iterator to the first inserted element or pos if first == last. - //! - //! Throws: If memory allocation throws, T's constructor from a - //! dereferenced InpIt throws or T's copy/move constructor/assignment throws. - //! - //! Complexity: Linear to boost::container::iterator_distance [first, last). - template - iterator insert( const_iterator const position, InIt const first, InIt const last ) - { - auto const n{ static_cast( std::distance( first, last ) ) }; - auto const iter{ make_space_for_insert( position, n ) }; - std::copy_n( first, n, iter ); - return iter; - } - - //! Requires: position must be a valid iterator of *this. - //! - //! Effects: Insert a copy of the [il.begin(), il.end()) range before position. - //! - //! Returns: an iterator to the first inserted element or position if first == last. - //! - //! Complexity: Linear to the range [il.begin(), il.end()). - iterator insert( const_iterator const position, std::initializer_list const il ) - { - return insert( position, il.begin(), il.end() ); - } - - void append_range( std::ranges::range auto && __restrict rng ) - { - auto const current_size{ size() }; - storage_.expand( to_byte_sz( current_size + std::size( rng ) ) ); - std::uninitialized_copy( std::begin( rng ), std::end( rng ), nth( current_size ) ); - } - void append_range( std::initializer_list< value_type > const rng ) { append_range( std::span{ rng.begin(), rng.end() } ); } - - //! Effects: Removes the last element from the container. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant time. - void pop_back() noexcept - { - BOOST_ASSERT(!this->empty()); - std::destroy_at( &back() ); - shrink_storage_to( size() - 1 ); - } - - //! Effects: Erases the element at position pos. - //! - //! Throws: Nothing. - //! - //! Complexity: Linear to the elements between pos and the - //! last element. Constant if pos is the last element. - iterator erase( const_iterator const position ) noexcept - { - verify_iterator( position ); - auto const pos_index{ index_of( position ) }; - auto const mutable_pos{ nth( pos_index ) }; - std::move( mutable_pos + 1, end(), mutable_pos ); - pop_back(); - return nth( pos_index ); - } - - //! Effects: Erases the elements pointed by [first, last). - //! - //! Throws: Nothing. - //! - //! Complexity: Linear to the distance between first and last - //! plus linear to the elements between pos and the last element. - iterator erase( const_iterator const first, const_iterator const last ) noexcept - { - verify_iterator( first ); - verify_iterator( last ); - BOOST_ASSERT( first <= last ); - auto const first_index{ index_of( first ) }; - if ( first != last ) [[ likely ]] - { - auto const mutable_start{ nth( first_index ) }; - auto const mutable_end { nth( index_of( last ) ) }; - auto const new_end { std::move( mutable_end, end(), mutable_start ) }; - for ( auto & element : { new_end, end() } ) - std::destroy_at( &element ); - shrink_storage_to( static_cast( new_end - begin() ) ); - } - return nth( first_index ); - } - - //! Effects: Swaps the contents of *this and x. - //! - //! Throws: Nothing. - //! - //! Complexity: Constant. - void swap( vector & x ) noexcept { using std::swap; swap( this->storage_, x.storage_ ); } - - //! Effects: Erases all the elements of the vector. - //! - //! Throws: Nothing. - //! - //! Complexity: Linear to the number of elements in the container. - void clear() noexcept - { - for ( auto & element : span() ) - std::destroy_at( &element ); - shrink_storage_to( 0 ); - } - - //! Effects: Returns true if x and y are equal - //! - //! Complexity: Linear to the number of elements in the container. - [[ nodiscard ]] friend bool operator==( vector const & x, vector const & y ) noexcept { return std::equal( x.begin(), x.end(), y.begin(), y.end() ); } + vector & operator=( vector && ) = default; + vector & operator=( vector const & ) = default; - //! Effects: Returns true if x and y are unequal - //! - //! Complexity: Linear to the number of elements in the container. - [[ nodiscard ]] friend bool operator!=(const vector& x, const vector& y) noexcept { return !(x == y); } - - //! Effects: x.swap(y) - //! - //! Complexity: Constant. - friend void swap( vector & x, vector & y ) noexcept { x.swap( y ); } - - /////////////////////////////////////////////////////////////////////////// - // Extensions - /////////////////////////////////////////////////////////////////////////// - - auto map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_file( file, policy ); } + auto map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { BOOST_ASSUME( !storage_t::has_attached_storage() ); return storage_t::map_file( file, policy ); } template - auto map_memory( size_type const initial_size = 0, InitPolicy init_policy = {} ) noexcept + auto map_memory( sz_t const initial_size = 0, InitPolicy init_policy = {} ) noexcept { - BOOST_ASSERT( !has_attached_storage() ); - auto result{ storage_.map_memory( to_byte_sz( initial_size ) )() }; + BOOST_ASSUME( !storage_t::has_attached_storage() ); + auto result{ storage_t::map_memory( storage_t::to_byte_sz( initial_size ) )() }; if ( std::is_same_v && initial_size && result ) { - std::uninitialized_default_construct( begin(), end() ); + std::uninitialized_default_construct( impl::begin(), impl::end() ); } return result.as_fallible_result(); } - bool has_attached_storage() const noexcept { return static_cast( storage_ ); } - - bool file_backed() const noexcept { return storage_.file_backed(); } - - void close() noexcept { storage_.close(); } - - //! Effects: If n is less than or equal to capacity(), this call has no - //! effect. Otherwise, it is a request for allocation of additional memory - //! (memory expansion) that will not invalidate iterators. - //! If the request is successful, then capacity() is greater than or equal to - //! n; otherwise, capacity() is unchanged. In either case, size() is unchanged. + //! Effects: Returns a copy of the internal allocator. //! - //! Throws: If memory allocation throws or T's copy/move constructor throws. + //! Throws: If allocator's copy constructor throws. //! - //! Note: Non-standard extension. - bool stable_reserve( size_type new_cap ) = /*TODO*/ delete; - - void grow_to( size_type const target_size, default_init_t ) requires( std::is_trivially_default_constructible_v ) - { - storage_.expand( to_byte_sz( target_size ) ); - } - PSI_WARNING_DISABLE_PUSH() - PSI_WARNING_MSVC_DISABLE( 4702 ) // unreachable code - void grow_to( size_type const target_size, value_init_t ) - { - auto const current_size{ size() }; - BOOST_ASSUME( target_size >= current_size ); - storage_.expand( to_byte_sz( target_size ) ); - if constexpr ( std::is_trivially_constructible_v ) - { - [[ maybe_unused ]] auto const newSpaceBegin{ reinterpret_cast( data() + current_size ) }; - [[ maybe_unused ]] auto const newSpaceSize { ( target_size - current_size ) * sizeof( value_type ) }; -# if 0 - std::memset( newSpaceBegin, 0, newSpaceSize ); -# else - BOOST_ASSERT_MSG - ( - *std::max_element( newSpaceBegin, newSpaceBegin + newSpaceSize ) == 0, - "Expecting files to be zero-extended" - ); -# endif - } - else - { - std::uninitialized_default_construct( nth( current_size ), end() ); - } - } - PSI_WARNING_DISABLE_POP() - void grow_by( size_type const delta, auto const init_policy ) - { - grow_to( size() + delta, init_policy ); - } - - void shrink_to( size_type const target_size ) noexcept - { - auto const current_size{ size() }; - BOOST_ASSUME( target_size <= current_size ); - if ( !std::is_trivially_destructible_v ) - { - for ( auto & element : span().subspan( target_size ) ) - std::destroy_at( &element ); - } - storage_.shrink( to_byte_sz( target_size ) ); - } - void shrink_by( size_type const delta ) noexcept - { - shrink_to( size() - delta ); - } - - decltype( auto ) user_header_data() noexcept { return storage_.header_storage(); } - - // helper getter for generic code that wishes to do basic manipulation on - // vm::vectors w/o being templated (contiguous_container_storage_base does not - // publicize functionality that could be used to make it out of sync with the - // corresponding vm::vector) - contiguous_container_storage_base & storage_base() noexcept { return storage_; } - - void flush_async ( std::size_t const byteOffset, std::size_t const byteSize ) const noexcept { storage_.flush_async ( byteOffset, byteSize ); } - void flush_blocking( std::size_t const byteOffset, std::size_t const byteSize ) const noexcept { storage_.flush_blocking( byteOffset, byteSize ); } - - void flush_async () const noexcept { flush_async ( 0, storage_.size() ); } - void flush_blocking() const noexcept { flush_blocking( 0, storage_.size() ); } - -private: - PSI_WARNING_DISABLE_PUSH() - PSI_WARNING_GCC_OR_CLANG_DISABLE( -Wsign-conversion ) - static T * to_t_ptr ( mapped_view::value_type * const ptr ) noexcept { return reinterpret_cast( ptr ); } - static sz_t to_t_sz ( auto const byte_sz ) noexcept { BOOST_ASSUME( byte_sz % sizeof( T ) == 0 ); return static_cast( byte_sz / sizeof( T ) ); } - static sz_t to_byte_sz( auto const sz ) noexcept - { - auto const rez{ sz * sizeof( T ) }; - BOOST_ASSERT( rez <= std::numeric_limits::max() ); - return static_cast( rez ); - } - PSI_WARNING_DISABLE_POP() - - void shrink_storage_to( size_type const target_size ) noexcept - { - storage_.shrink( to_byte_sz( target_size ) ); - } - - void verify_iterator( [[ maybe_unused ]] const_iterator const iter ) const noexcept - { - BOOST_ASSERT( iter >= begin() ); - BOOST_ASSERT( iter <= end () ); - } - - iterator make_space_for_insert( const_iterator const position, size_type const n ) - { - verify_iterator( position ); - auto const position_index{ index_of( position ) }; - auto const current_size { size() }; - auto const new_size { current_size + n }; - storage_.expand( to_byte_sz( new_size ) ); - auto const elements_to_move{ static_cast( current_size - position_index ) }; - if constexpr ( is_trivially_moveable ) - { - std::move( nth( position_index ), nth( position_index + elements_to_move ), nth( position_index + n ) ); - } - else // future support for generic types - { - auto const elements_to_move_to_uninitialized_space{ n }; - auto const elements_to_move_to_the_current_end { static_cast( elements_to_move - elements_to_move_to_uninitialized_space ) }; - std::uninitialized_move_n( nth( current_size - elements_to_move_to_uninitialized_space ), elements_to_move_to_uninitialized_space, nth( current_size ) ); - std::move ( nth( position_index ), nth( position_index + elements_to_move_to_the_current_end ) , nth( position_index + n ) ); - } - return nth( position_index ); - } + //! Complexity: Constant. + storage_t const & get_stored_allocator() const noexcept { return *this; } }; // class vector - -template -#if defined( _MSC_VER ) && !defined( NDEBUG ) -// Boost.Container flat_set is bugged: tries to dereference end iterators - asserts at runtime with secure/checked STL/containers -// https://github.com/boostorg/container/issues/261 -requires is_trivially_moveable -class unchecked_vector : public vector -{ -public: - using base = vector; - - using iterator = typename base:: pointer; - using const_iterator = typename base::const_pointer; - - using base::base; - - [[ nodiscard ]] auto begin() noexcept { return base::data(); } - [[ nodiscard ]] auto begin() const noexcept { return base::data(); } - [[ nodiscard ]] auto cbegin() const noexcept { return begin(); } - - [[ nodiscard ]] auto end() noexcept { return begin() + base::size(); } - [[ nodiscard ]] auto end() const noexcept { return begin() + base::size(); } - [[ nodiscard ]] auto cend() const noexcept { return end(); } - - iterator erase( const_iterator const first, const_iterator const last ) noexcept - { - return - begin() - + - base::index_of - ( - base::erase - ( - base::nth( static_cast( first - cbegin() ) ), - base::nth( static_cast( last - cbegin() ) ) - ) - ); - } - - iterator erase( const_iterator const position ) noexcept - { - return - begin() - + - base::index_of - ( - base::erase - ( - base::cbegin() + ( position - cbegin() ) - ) - ); - } - - template < typename ... Args > - iterator insert( const_iterator const position, Args &&... args ) noexcept - { - return - begin() - + - base::index_of - ( - base::insert - ( - base::nth( static_cast( position - begin() ) ), - std::forward< Args >( args )... - ) - ); - } -}; -#else -using unchecked_vector = vector; -#endif - -template -using unchecked = unchecked_vector; - //------------------------------------------------------------------------------ } // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/vector_impl.hpp b/include/psi/vm/containers/vector_impl.hpp new file mode 100644 index 0000000..2b168fa --- /dev/null +++ b/include/psi/vm/containers/vector_impl.hpp @@ -0,0 +1,878 @@ +//////////////////////////////////////////////////////////////////////////////// +/// Base CRTP-like implementation of shared, standard C++latest functionality +/// for vector-like containers. Also provides extensions like default vs value +/// initialization and explicit grow and shrink (noexcept) vs resize methods, +/// stable/try-expansion (TODO), checked iterators, configurable size_type, +/// pass-by-value pass-in-reg ABI for supported trivial types... +/// w/ special emphasis on code reuse and bloat reduction. +/// (requires deducing this support) +/// TODO extract the entire container subdirectory into a separate library +/// e.g. https://github.com/AmadeusITGroup/amc +//////////////////////////////////////////////////////////////////////////////// +/// +/// Copyright (c) Domagoj Saric. +/// +/// Use, modification and distribution is subject to the +/// Boost Software License, Version 1.0. +/// (See accompanying file LICENSE_1_0.txt or copy at +/// http://www.boost.org/LICENSE_1_0.txt) +/// +/// For more information, see http://www.boost.org +/// +//////////////////////////////////////////////////////////////////////////////// +//------------------------------------------------------------------------------ +#pragma once + +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +namespace detail +{ + [[ noreturn, gnu::cold ]] void throw_out_of_range(); + [[ noreturn, gnu::cold ]] void throw_bad_alloc (); + + template + constexpr T * mutable_iter( T const * const ptr ) noexcept { return const_cast( ptr ); } + + template + constexpr Base mutable_iter( std::basic_const_iterator const iter ) noexcept { return iter.base(); } +#if defined( _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR ) + template + auto mutable_iter( std::__bounded_iter const iter ) noexcept { return reinterpret_cast const &>( iter ); } +#elif _ITERATOR_DEBUG_LEVEL + template + auto mutable_iter( std::_Span_iterator const iter ) noexcept { return reinterpret_cast const &>( iter ); } +#endif + + struct init_policy_tag{}; +} // namespace detail + +template +[[ gnu::const ]] constexpr Target verified_cast( std::unsigned_integral auto const source ) noexcept +{ + auto constexpr target_max{ std::numeric_limits::max() }; + BOOST_ASSUME( source <= target_max ); + auto const result{ static_cast( source ) }; + BOOST_ASSUME( result == source ); + return result; +} + +struct no_init_t : detail::init_policy_tag{}; inline constexpr no_init_t no_init ; +struct default_init_t : detail::init_policy_tag{}; inline constexpr default_init_t default_init; +struct value_init_t : detail::init_policy_tag{}; inline constexpr value_init_t value_init ; + +template +concept init_policy = std::is_base_of_v; + +// see the note for up() on why the Impl parameter is used/required +template +class [[ nodiscard, clang::trivial_abi ]] vector_impl +{ +public: + using value_type = T; + using pointer = value_type *; + using const_pointer = value_type const *; + using reference = value_type &; + using const_reference = value_type const &; + using param_const_ref = std::conditional_t, value_type const, pass_in_reg>; + using size_type = sz_t; + using difference_type = std::make_signed_t; +#if defined( _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR ) + using iterator = std::__bounded_iter; + using const_iterator = std::basic_const_iterator; // __bounded_iter is not convertible to __bounded_iter so we have to use the std wrapper +#elif _ITERATOR_DEBUG_LEVEL + // checked_array_iterator is deprecated so use the span iterator wrapper + using iterator = std::_Span_iterator; + using const_iterator = std::basic_const_iterator; // same as for __bounded_iter +#else + using iterator = pointer; + using const_iterator = const_pointer; +#endif + using reverse_iterator = std::reverse_iterator< iterator>; + using const_reverse_iterator = std::reverse_iterator; + +private: + // Workaround for a deducing this defect WRT to private inheritance: the + // problem&solution described in the paper do not cover this case - where + // * the base class is empty and + // * the implementation details are in an immediately derived class D0 + // * there is class Dx which then privately inherits from D0 and calls + // vector_impl methods + // - then the type of the self argument gets deduced as Dx (const &) - + // through which members of D0 cannot be accessed due to private + // inheritance. Moreover in that case we do not know of the D0 type and + // cannot cast self to it - therefore in order to support this use case we + // have to require that 'actual implementation' derived type to be + // explicitly specified as template parameter. + // (Explicitly using the Impl type for self then has the added benefit of + // eliminating the extra compile-time and binary size hit of instantiating + // base/vector_impl methods for with all the possible derived types). + // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p0847r7.html#the-shadowing-mitigation-private-inheritance-problem + // https://stackoverflow.com/questions/844816/c-style-upcast-and-downcast-involving-private-inheritance (tunneling magic of C-style casts) + template [[ gnu::const ]] static constexpr Impl & up( U & self ) noexcept { static_assert( std::is_base_of_v ); return (Impl &)self; } + template [[ gnu::const ]] static constexpr Impl const & up( U const & self ) noexcept { return up( const_cast( self ) ); } + +private: + static constexpr bool moved_out_value_is_trivially_destructible{ true }; + + constexpr auto begin_ptr( this auto & self ) noexcept { return self.data(); } + constexpr auto end_ptr( this auto & self ) noexcept { return self.data() + self.size(); } + constexpr iterator make_iterator( [[ maybe_unused ]] this Impl & self, value_type * const ptr ) noexcept + { + return +# if defined( _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR ) + std::__make_bounded_iter( ptr, self.begin_ptr(), self.end_ptr() ); +# elif _ITERATOR_DEBUG_LEVEL + std::_Span_iterator{ ptr, self.begin_ptr(), self.end_ptr() }; +# else + ptr; +# endif + } + constexpr iterator make_iterator( this Impl & self, size_type const offset ) noexcept + { + auto const begin{ self.data() }; + return self.make_iterator( begin + offset ); + } + constexpr const_iterator make_iterator( this Impl const & self, size_type const offset ) noexcept + { + return { const_cast( self ).make_iterator( offset ) }; + } + // constructor helper - for initializing the derived Impl class - only to be + // used in constructors! + constexpr Impl & initialized_impl( size_type const initial_size, no_init_t ) noexcept + { + auto & impl{ static_cast( *this ) }; + impl.storage_init( initial_size ); + BOOST_ASSUME( impl.size() == initial_size ); + return impl; + } + constexpr Impl & initialized_impl( size_type const initial_size, default_init_t ) noexcept + { + auto & impl{ initialized_impl( initial_size, no_init ) }; + std::uninitialized_default_construct_n( impl.data(), impl.size() ); + return impl; + } + constexpr Impl & initialized_impl( size_type const initial_size, value_init_t ) noexcept + { + auto & impl{ initialized_impl( initial_size, no_init ) }; + std::uninitialized_value_construct_n( impl.data(), impl.size() ); + return impl; + } + + constexpr vector_impl( Impl & self, Impl const & other ) noexcept( noexcept_storage() && std::is_nothrow_copy_constructible_v ) + { + BOOST_ASSUME( self.empty() ); + auto const sz{ other.size() }; + auto const self_data{ self.grow_to( sz, no_init ) }; + std::uninitialized_copy_n( other.begin(), sz, self_data ); + } + +protected: + constexpr vector_impl( ) noexcept = default; + constexpr vector_impl( vector_impl const & ) noexcept = default; + constexpr vector_impl( vector_impl && ) noexcept = default; + constexpr ~vector_impl( ) noexcept = default; + +public: + // MSVC (VS 17.12.3) fails compilation if this is a variable + static bool consteval noexcept_storage() { return noexcept( std::declval().storage_grow_to( size_type( 123 ) ) ); }; + + // non standard default: default-initialization + constexpr explicit vector_impl( size_type const initial_size ) noexcept( noexcept_storage() && std::is_nothrow_default_constructible_v ) : vector_impl( initial_size, default_init ) {} + constexpr vector_impl( size_type const initial_size, init_policy auto const policy ) noexcept( noexcept_storage() && std::is_nothrow_default_constructible_v ) + { + initialized_impl( initial_size, policy ); + } + + constexpr vector_impl( size_type const count, param_const_ref const value ) noexcept( noexcept_storage() && std::is_nothrow_copy_constructible_v ) + { + auto & impl{ initialized_impl( count, no_init ) }; + std::uninitialized_fill_n( impl.data(), count, value ); + BOOST_ASSUME( impl.size() == count ); + } + + template + constexpr vector_impl( It const first, It const last ) noexcept( noexcept_storage() && std::is_nothrow_copy_constructible_v ) + { + if constexpr ( std::random_access_iterator ) + { + auto const sz{ std::distance( first, last ) }; + auto & impl{ initialized_impl( sz, no_init ) }; + // STL utility functions handle EH safety - no need to catch to + // reset size as Impl/the derived class should not attempt cleanup + // if this (its base constructor) fails. + std::uninitialized_copy_n( first, sz, impl.data() ); + } + else + { + auto & impl{ initialized_impl( 0, no_init ) }; + std::copy( first, last, std::back_inserter( impl ) ); + } + } + + constexpr vector_impl( std::initializer_list const initial_values ) noexcept( noexcept_storage() && std::is_nothrow_copy_constructible_v ) + : vector_impl( initial_values.begin(), initial_values.end() ) + {} + + + constexpr vector_impl & operator=( vector_impl && ) noexcept = default; + constexpr vector_impl & operator=( this Impl & self, Impl const & other ) noexcept( noexcept_storage() && std::is_nothrow_copy_constructible_v ) + { + BOOST_ASSUME( &self != &other ); // not going to support self assignment + self.assign( other ); + return self; + } + + constexpr vector_impl & operator=( this Impl & self, std::initializer_list const data ) { self.assign( data ); return self; } + + //! Effects: Assigns the the range [first, last) to *this. + //! + //! Throws: If memory allocation throws or T's copy/move constructor/assignment or + //! T's constructor/assignment from dereferencing InpIt throws. + //! + //! Complexity: Linear to n. + template + void assign( this Impl & self, It first, It const last ) + requires( !std::is_trivially_destructible_v ) + { + // Overwrite all elements we can from [first, last) + auto cur { self.begin() }; + auto const end_it{ self.end () }; + if constexpr ( std::random_access_iterator ) { + auto const overwrite_size{ std::min( self.size(), std::distance( first, last ) ) }; + std::tie( first, cur ) = std::ranges::copy_n( first, overwrite_size, cur ); + } else { + while ( ( first != last ) && ( cur != end_it ) ) + *cur++ = *first++; + } + + if ( first == last ) + { + // There are no more elements in the sequence, erase remaining + auto const target_size{ static_cast( cur - self.begin() ) }; + std::destroy( cur, end_it ); + self.shrink_storage_to( target_size ); + } + else + { + // There are more elements in the range, insert the remaining ones + self.append_range( std::ranges::subrange( first, last ) ); + } + } + + template + void assign( this Impl & self, It const first, It const last ) + requires( std::is_trivially_destructible_v ) + { + auto const input_size{ static_cast( std::distance( first, last ) ) }; + self.resize( input_size, no_init ); + std::uninitialized_copy_n( first, input_size, self.begin() ); + } + template + void assign( this Impl & self, Rng && data ) + { + static_assert( std::ranges::range> ); + if constexpr ( std::is_rvalue_reference_v ) + self.assign( std::make_move_iterator( data.begin() ), std::make_move_iterator( data.end() ) ); + else + self.assign( data.begin(), data.end() ); + } + void assign( this Impl & self, Impl && other ) noexcept( std::is_nothrow_move_assignable_v ) + { + self = std::move( other ); + } + + //! Effects: Assigns the n copies of val to *this. + //! + //! Throws: If memory allocation throws or + //! T's copy/move constructor/assignment throws. + //! + //! Complexity: Linear to n. + void assign( this Impl & self, size_type const n, param_const_ref val ) = /*TODO*/ delete; + + ////////////////////////////////////////////// + // + // iterators + // + ////////////////////////////////////////////// + + //! Effects: Returns an iterator to the first element contained in the vector. + //! Throws: Nothing. + //! Complexity: Constant. + [[ nodiscard ]] auto begin( this auto & self ) noexcept { return up( self ).make_iterator( size_type{ 0 } ); } + [[ nodiscard ]] const_iterator cbegin( this Impl const & self ) noexcept { return self.begin(); } + + //! Effects: Returns an iterator to the end of the vector. + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + [[ nodiscard ]] auto end( this auto & self ) noexcept { return up( self ).make_iterator( self.size() ); } + [[ nodiscard ]] const_iterator cend( this Impl const & self ) noexcept { return self.end(); } + + //! Effects: Returns a reverse_iterator pointing to the beginning + //! of the reversed vector. + //! Throws: Nothing. + //! Complexity: Constant. + [[ nodiscard ]] auto rbegin( this auto & self ) noexcept { return std::make_reverse_iterator( self.begin() ); } + [[ nodiscard ]] const_reverse_iterator crbegin( this Impl const & self ) noexcept { return self.rbegin(); } + + //! Effects: Returns a reverse_iterator pointing to the end + //! of the reversed vector. + //! Throws: Nothing. + //! Complexity: Constant. + [[ nodiscard ]] auto rend( this auto & self ) noexcept { return std::make_reverse_iterator( self.end() ); } + [[ nodiscard ]] const_reverse_iterator crend( this Impl const & self ) noexcept { return self.rend(); } + + ////////////////////////////////////////////// + // + // capacity + // + ////////////////////////////////////////////// + + //! Effects: Returns true if the vector contains no elements. + //! Throws: Nothing. + //! Complexity: Constant. + [[ nodiscard, gnu::pure ]] bool empty( this Impl const & self ) noexcept { return BOOST_UNLIKELY( self.size() == 0 ); } + + //! Effects: Returns the largest possible size of the vector. + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + [[ nodiscard ]] static constexpr size_type max_size() noexcept { return static_cast( std::numeric_limits::max() / sizeof( value_type ) ); } + + void resize( this Impl & self, size_type const new_size, auto const init_policy ) + { + if ( new_size > self.size() ) self. grow_to( new_size, init_policy ); + else self.shrink_to( new_size ); + } + // intentional non-standard behaviour: default_init by default + void resize( this Impl & self, size_type const new_size ) { self.resize( new_size, default_init ); } + + //! Effects: Inserts or erases elements at the end such that + //! the size becomes n. New elements are copy constructed from x. + //! + //! Throws: If memory allocation throws, or T's copy/move constructor throws. + //! + //! Complexity: Linear to the difference between size() and new_size. + void resize( this Impl & self, size_type const new_size, param_const_ref x ) + { + auto const current_size{ self.size() }; + BOOST_ASSUME( new_size >= current_size ); + self.resize( new_size, default_init ); + auto uninitialized_span{ self.span().subspan( current_size ) }; + std::uninitialized_fill( uninitialized_span.begin(), uninitialized_span.end(), x ); + } + + void shrink_to_fit( this Impl & self ) noexcept { self.storage_shrink_to( self.size() ); } + + ////////////////////////////////////////////// + // + // element access + // + ////////////////////////////////////////////// + + //! Requires: !empty() + //! + //! Effects: Returns a reference to the first + //! element of the container. + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + [[ nodiscard ]] auto & front( this auto & self ) noexcept { return up( self ).span().front(); } + + //! Requires: !empty() + //! + //! Effects: Returns a reference to the last + //! element of the container. + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + [[ nodiscard ]] auto & back( this auto & self ) noexcept { return up( self ).span().back(); } + + //! Requires: size() > n. + //! + //! Effects: Returns a reference to the nth element + //! from the beginning of the container. + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + [[ nodiscard ]] auto & operator[]( this auto & self, size_type const n ) noexcept { return up( self ).span()[ n ]; } + + //! Requires: size() >= n. + //! + //! Effects: Returns an iterator to the nth element + //! from the beginning of the container. Returns end() + //! if n == size(). + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + //! + //! Note: Non-standard extension + [[ nodiscard ]] auto nth( this auto & self, size_type const n ) noexcept + { + BOOST_ASSUME( n <= self.size() ); + return self.begin() + static_cast( n ); + } + + //! Requires: begin() <= p <= end(). + //! + //! Effects: Returns the index of the element pointed by p + //! and size() if p == end(). + //! + //! Throws: Nothing. + //! + //! Complexity: Constant. + //! + //! Note: Non-standard extension + [[ nodiscard ]] size_type index_of( this Impl const & self, const_iterator const p ) noexcept + { + self.verify_iterator( p ); + return static_cast( p - self.begin() ); + } + + //! Requires: size() > n. + //! + //! Effects: Returns a reference to the nth element + //! from the beginning of the container. + //! + //! Throws: range_error if n >= size() + //! + //! Complexity: Constant. + [[ nodiscard ]] auto & at( this auto & self, size_type const n ) + { +# if __cpp_lib_span >= 202311L + return self.span().at( n ); +# else + if ( n >= self.size() ) + detail::throw_out_of_range(); + return self[ n ]; +# endif + } + + ////////////////////////////////////////////// + // + // data access + // + ////////////////////////////////////////////// + + template + [[ nodiscard, gnu::pure ]] const_pointer data( this Self const & self ) noexcept { return const_cast( self ).Self::data(); } + + [[ nodiscard, gnu::pure ]] auto span( this auto & self ) noexcept { return std::span{ self.data(), self.size() }; } + + ////////////////////////////////////////////// + // + // modifiers + // + ////////////////////////////////////////////// + + //! Effects: Inserts an object of type T constructed with + //! std::forward(args)... at the end of the vector. + //! + //! Returns: A reference to the created object. + //! + //! Throws: If memory allocation throws or the in-place constructor throws or + //! T's copy/move constructor throws. + //! + //! Complexity: Amortized constant time. + + template + static reference construct_at( value_type & placeholder, Args &&...args ) noexcept( std::is_nothrow_constructible_v ) + { + return *std::construct_at( &placeholder, std::forward( args )... ); + } + static reference construct_at( value_type & placeholder ) noexcept( std::is_nothrow_default_constructible_v ) + { + return *(new (&placeholder) value_type); // default to default init + } + + template + reference emplace_back( this Impl & self, Args &&...args ) + { + auto const current_size{ self.size() }; + if constexpr ( sizeof...( Args ) || true ) // grow_by( 1, default_init ) would call uninitialized_default_construct (i.e. a loop) - TODO grow_by_one and shrink_by_one methods for push&pop-back operations + { + auto const data{ self.grow_by( 1, no_init ) }; + auto & placeholder{ data[ current_size ] }; + try { + return construct_at( placeholder, std::forward( args )... ); + } catch( ... ) { + self.shrink_by( 1 ); + throw; + } + } + else + { + return self.grow_by( 1, default_init )[ current_size ]; + } + } + + //! Effects: Inserts an object of type T constructed with + //! std::forward(args)... in the end of the vector. + //! + //! Throws: If the in-place constructor throws. + //! + //! Complexity: Constant time. + //! + //! Note: Non-standard extension. + template + bool stable_emplace_back( this Impl & self, Args &&... args ) = /*TODO*/delete; + + //! Requires: position must be a valid iterator of *this. + //! + //! Effects: Inserts an object of type T constructed with + //! std::forward(args)... before position + //! + //! Throws: If memory allocation throws or the in-place constructor throws or + //! T's copy/move constructor/assignment throws. + //! + //! Complexity: If position is end(), amortized constant time + //! Linear time otherwise. + template + iterator emplace( this Impl & self, const_iterator const position, Args &&... args ) + { + auto const iter{ self.make_space_for_insert( position, 1 ) }; + if constexpr ( moved_out_value_is_trivially_destructible ) + construct_at( *iter, std::forward( args )... ); + else + *iter = { std::forward( args )... }; + return iter; + } + + //! Effects: Inserts a copy of x at the end of the vector. + //! + //! Throws: If memory allocation throws or + //! T's copy/move constructor throws. + //! + //! Complexity: Amortized constant time. + void push_back( this Impl & self, param_const_ref x ) noexcept( noexcept_storage() && std::is_nothrow_copy_constructible_v ) + { + self.emplace_back( x ); + } + + //! Effects: Constructs a new element in the end of the vector + //! and moves the resources of x to this new element. + //! + //! Throws: If memory allocation throws or + //! T's copy/move constructor throws. + //! + //! Complexity: Amortized constant time. + void push_back( this Impl & self, value_type && x ) noexcept( noexcept_storage() && std::is_nothrow_move_constructible_v ) + requires( !std::is_trivial_v ) // otherwise better to go through the pass-in-reg overload + { + self.emplace_back( std::move( x ) ); + } + + //! Requires: position must be a valid iterator of *this. + //! + //! Effects: Insert a copy of x before position. + //! + //! Throws: If memory allocation throws or T's copy/move constructor/assignment throws. + //! + //! Complexity: If position is end(), amortized constant time + //! Linear time otherwise. + iterator insert( this Impl & self, const_iterator const position, param_const_ref x ) { return self.emplace( position, x ); } + + //! Requires: position must be a valid iterator of *this. + //! + //! Effects: Insert a new element before position with x's resources. + //! + //! Throws: If memory allocation throws. + //! + //! Complexity: If position is end(), amortized constant time + //! Linear time otherwise. + iterator insert( this Impl & self, const_iterator const position, value_type && x ) requires( !std::is_trivial_v ) { return self.emplace( position, std::move( x ) ); } + + //! Requires: position must be a valid iterator of *this. + //! + //! Effects: Insert n copies of x before pos. + //! + //! Returns: an iterator to the first inserted element or p if n is 0. + //! + //! Throws: If memory allocation throws or T's copy/move constructor throws. + //! + //! Complexity: Linear to n. + iterator insert( this Impl & self, const_iterator const position, size_type const n, param_const_ref x ) + { + auto const iter{ self.make_space_for_insert( position, n ) }; + if constexpr ( moved_out_value_is_trivially_destructible ) + std::uninitialized_fill_n( iter, n, x ); + else + std::fill_n( iter, n, x ); + return iter; + } + + //! Requires: position must be a valid iterator of *this. + //! + //! Effects: Insert a copy of the [first, last) range before pos. + //! + //! Returns: an iterator to the first inserted element or pos if first == last. + //! + //! Throws: If memory allocation throws, T's constructor from a + //! dereferenced InpIt throws or T's copy/move constructor/assignment throws. + //! + //! Complexity: Linear to boost::container::iterator_distance [first, last). + template + iterator insert( this Impl & self, const_iterator const position, InIt const first, InIt const last ) + { + auto const n{ static_cast( std::distance( first, last ) ) }; + auto const iter{ self.make_space_for_insert( position, n ) }; + if constexpr ( moved_out_value_is_trivially_destructible ) + std::uninitialized_copy_n( first, n, iter ); + else + std::copy_n( first, n, iter ); + return iter; + } + + //! Requires: position must be a valid iterator of *this. + //! + //! Effects: Insert a copy of the [il.begin(), il.end()) range before position. + //! + //! Returns: an iterator to the first inserted element or position if first == last. + //! + //! Complexity: Linear to the range [il.begin(), il.end()). + iterator insert( this Impl & self, const_iterator const position, std::initializer_list const il ) + { + return self.insert( position, il.begin(), il.end() ); + } + + template + void append_range( this Impl & self, Rng && __restrict rng ) + { + auto const current_size{ self.size() }; + auto const additional_size{ verified_cast( std:: size( rng ) ) }; + auto const input_begin { std::begin( rng ) }; + auto const target_position{ self.grow_by( additional_size, no_init ) + current_size }; + try + { + if constexpr ( std::is_rvalue_reference_v ) + std::uninitialized_move_n( input_begin, additional_size, target_position ); + else + std::uninitialized_copy_n( input_begin, additional_size, target_position ); + } + catch (...) + { + self.storage_shrink_size_to( current_size ); + throw; + } + } +#ifndef __cpp_lib_span_initializer_list + void append_range( this Impl & self, std::initializer_list const rng ) { self.append_range( std::span{ rng.begin(), rng.end() } ); } +#endif + + //! Effects: Removes the last element from the container. + //! + //! Throws: Nothing. + //! + //! Complexity: Constant time. + void pop_back( this Impl & self ) noexcept + { + BOOST_ASSUME( !self.empty() ); + std::destroy_at( &self.back() ); + self.storage_dec_size(); + } + + //! Effects: Erases the element at position pos. + //! + //! Throws: Nothing. + //! + //! Complexity: Linear to the elements between pos and the + //! last element. Constant if pos is the last element. + iterator erase( this Impl & self, const_iterator const position ) noexcept + { + self.verify_iterator( position ); + auto const pos_index{ self.index_of( position ) }; + auto const mutable_pos{ self.nth( pos_index ) }; + std::shift_left( mutable_pos, self.end(), 1 ); + if constexpr ( moved_out_value_is_trivially_destructible ) + self.storage_dec_size(); + else + self.pop_back(); + return self.nth( pos_index ); + } + + //! Effects: Erases the elements pointed by [first, last). + //! + //! Throws: Nothing. + //! + //! Complexity: Linear to the distance between first and last + //! plus linear to the elements between pos and the last element. + iterator erase( this Impl & self, const_iterator const first, const_iterator const last ) noexcept + { + self.verify_iterator( first ); + self.verify_iterator( last ); + BOOST_ASSERT( first <= last ); + auto const first_index{ self.index_of( first ) }; + auto const mutable_start{ detail::mutable_iter( first ) }; + auto const mutable_end { detail::mutable_iter( last ) }; + auto const new_end { std::move( mutable_end, self.end(), mutable_start ) }; + auto const new_size { static_cast( new_end - self.begin() ) }; + if constexpr ( moved_out_value_is_trivially_destructible ) + self.storage_shrink_size_to( new_size ); + else + self.shrink_to( new_size ); + return self.nth( first_index ); + } + //! Effects: Erases all the elements of the vector. + //! + //! Throws: Nothing. + //! + //! Complexity: Linear to the number of elements in the container. + void clear( this Impl & self ) noexcept + { + std::destroy( self.begin(), self.end() ); + self.free(); + } + + //! Effects: Returns true if x and y are equal + //! + //! Complexity: Linear to the number of elements in the container. + [[ nodiscard ]] bool operator==( this auto const & self, std::ranges::range auto const & other ) noexcept { return std::equal( self.begin(), self.end(), other.begin(), other.end() ); } + + //! Effects: Returns true if x and y are unequal + //! + //! Complexity: Linear to the number of elements in the container. + [[ nodiscard ]] bool operator!=( this auto const & self, std::ranges::range auto const & other ) noexcept { return !(self == other); } + + void swap( this auto & self, auto & other ) noexcept { std::swap( self, other ); } + + + /////////////////////////////////////////////////////////////////////////// + // Extensions + /////////////////////////////////////////////////////////////////////////// + + //! Effects: If n is less than or equal to capacity(), this call has no + //! effect. Otherwise, it is a request for allocation of additional memory + //! (memory expansion) that will not invalidate iterators. + //! If the request is successful, then capacity() is greater than or equal to + //! n; otherwise, capacity() is unchanged. In either case, size() is unchanged. + //! + //! Throws: If memory allocation throws or T's copy/move constructor throws. + //! + //! Note: Non-standard extension. + bool stable_reserve( this Impl & self, size_type new_cap ) = /*TODO*/ delete; + + value_type * grow_to( this Impl & self, size_type const target_size, no_init_t ) { return self.storage_grow_to( target_size ); } + value_type * grow_to( this Impl & self, size_type const target_size, default_init_t ) + { + auto const current_size{ self.size() }; + auto const data{ self.grow_to( target_size, no_init ) }; + if constexpr ( !std::is_trivially_default_constructible_v ) { + try { + std::uninitialized_default_construct( &data[ current_size ], &data[ target_size ] ); + } catch(...) { + self.storage_shrink_size_to( current_size ); + throw; + } + } + return data; + } + + value_type * grow_to( this Impl & self, size_type const target_size, value_init_t ) + { + auto const current_size{ self.size() }; + BOOST_ASSUME( target_size >= current_size ); + auto const data{ self.grow_to( target_size, no_init ) }; + auto const uninitialized_space_begin{ &data[ current_size ] }; + auto const uninitialized_space_size { target_size - current_size }; + if constexpr ( std::is_trivially_constructible_v ) + { + auto const new_space_bytes { reinterpret_cast( data + current_size ) }; + auto const new_space_byte_size{ uninitialized_space_size * sizeof( value_type ) }; + if constexpr ( Impl::storage_zero_initialized ) + { + BOOST_ASSERT_MSG + ( + *std::max_element( new_space_bytes, new_space_bytes + new_space_byte_size ) == 0, + "Broken storage promise to zero-extend" + ); + } + else + { + std::memset( new_space_bytes, 0, new_space_byte_size ); + } + } + else + { + try { + std::uninitialized_value_construct_n( uninitialized_space_begin, uninitialized_space_size ); + } catch(...) { + self.storage_shrink_size_to( current_size ); + throw; + } + } + return data; + } + value_type * grow_by( this Impl & self, size_type const delta, auto const init_policy ) + { + return self.grow_to( self.size() + delta, init_policy ); + } + + void shrink_to( this Impl & self, size_type const target_size ) noexcept + { + BOOST_ASSUME( target_size <= self.size() ); + std::destroy( self.nth( target_size ), self.end() ); + self.storage_shrink_size_to( target_size ); // std::vector behaviour: never release/shrink capacity + } + void shrink_by( this Impl & self, size_type const delta ) noexcept { self.shrink_to( self.size() - delta ); } + +private: + void verify_iterator( [[ maybe_unused ]] this Impl const & self, [[ maybe_unused ]] const_iterator const iter ) noexcept + { + BOOST_ASSERT( iter >= self.begin() ); + BOOST_ASSERT( iter <= self.end () ); + } + + iterator make_space_for_insert( this Impl & self, const_iterator const position, size_type const n ) + { + self.verify_iterator( position ); + auto const position_index{ self.index_of( position ) }; + auto const current_size { self.size() }; + auto const new_size { current_size + n }; + auto const data{ self.grow_to( new_size, no_init ) }; + auto const elements_to_move{ static_cast( current_size - position_index ) }; + if constexpr ( is_trivially_moveable ) + { + std::uninitialized_move_n( &data[ position_index ], elements_to_move, &data[ position_index + n ] ); + } + else // future support for generic types + { + auto const elements_to_move_to_uninitialized_space{ n }; + auto const elements_to_move_to_the_current_end { static_cast( elements_to_move - elements_to_move_to_uninitialized_space ) }; + std::uninitialized_move_n( &data[ current_size - elements_to_move_to_uninitialized_space ], elements_to_move_to_uninitialized_space, &data[ current_size ] ); + std::move ( &data[ position_index ], &data[ position_index + elements_to_move_to_the_current_end ], &data[ position_index + n ] ); + } + return self.make_iterator( &data[ position_index ] ); + } +}; // class vector_impl + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/mapped_view/mapped_view.hpp b/include/psi/vm/mapped_view/mapped_view.hpp index ca844ed..1e93ac4 100644 --- a/include/psi/vm/mapped_view/mapped_view.hpp +++ b/include/psi/vm/mapped_view/mapped_view.hpp @@ -42,7 +42,7 @@ namespace psi::vm //////////////////////////////////////////////////////////////////////////////// template -class basic_mapped_view : public std::conditional_t +class [[ clang::trivial_abi ]] basic_mapped_view : public std::conditional_t { public: using error_t = error; @@ -89,6 +89,11 @@ class basic_mapped_view : public std::conditional_t map diff --git a/src/containers/vector.cpp b/src/containers/vector.cpp index 5196921..5045c97 100644 --- a/src/containers/vector.cpp +++ b/src/containers/vector.cpp @@ -23,7 +23,8 @@ namespace psi::vm namespace detail { - [[ noreturn ]] void throw_out_of_range() { throw std::out_of_range( "vm::vector access out of bounds" ); } + [[ noreturn, gnu::cold ]] void throw_out_of_range() { throw std::out_of_range( "vm::vector access out of bounds" ); } + [[ noreturn, gnu::cold ]] void throw_bad_alloc () { throw std::bad_alloc(); } } // namespace detail void contiguous_container_storage_base::close() noexcept @@ -35,7 +36,7 @@ void contiguous_container_storage_base::close() noexcept void contiguous_container_storage_base::flush_async ( std::size_t const beginning, std::size_t const size ) const noexcept { vm::flush_async ( mapped_span({ view_.subspan( beginning, size ) }) ); } void contiguous_container_storage_base::flush_blocking( std::size_t const beginning, std::size_t const size ) const noexcept { vm::flush_blocking( mapped_span({ view_.subspan( beginning, size ) }), mapping_.underlying_file() ); } -void contiguous_container_storage_base::expand( std::size_t const target_size ) +void * contiguous_container_storage_base::grow_to( std::size_t const target_size ) { BOOST_ASSUME( target_size > mapped_size() ); // basic (1.5x) geometric growth implementation @@ -47,16 +48,17 @@ void contiguous_container_storage_base::expand( std::size_t const target_size ) auto const new_capacity{ std::max( target_size, current_capacity * 3U / 2U ) }; set_size( mapping_, new_capacity ); } - expand_view( target_size ); + return expand_view( target_size ); } -void contiguous_container_storage_base::expand_view( std::size_t const target_size ) +void * contiguous_container_storage_base::expand_view( std::size_t const target_size ) { BOOST_ASSERT( get_size( mapping_ ) >= target_size ); view_.expand( target_size, mapping_ ); + return data(); } -void contiguous_container_storage_base::shrink( std::size_t const target_size ) noexcept +void * contiguous_container_storage_base::shrink_to( std::size_t const target_size ) noexcept( mapping::views_downsizeable ) { if constexpr ( mapping::views_downsizeable ) { @@ -69,6 +71,26 @@ void contiguous_container_storage_base::shrink( std::size_t const target_size ) set_size( mapping_, target_size )().assume_succeeded(); view_ = mapped_view::map( mapping_, 0, target_size ); } + return data(); +} + +void contiguous_container_storage_base::shrink_mapped_size_to( std::size_t const target_size ) noexcept( mapping::views_downsizeable ) +{ + if constexpr ( mapping::views_downsizeable ) + { + view_.shrink( target_size ); + } + else + { + view_.unmap(); + view_ = mapped_view::map( mapping_, 0, target_size ); + } +} + +void contiguous_container_storage_base::free() noexcept +{ + view_.unmap(); + set_size( mapping_, 0 )().assume_succeeded(); } void contiguous_container_storage_base::shrink_to_fit() noexcept @@ -76,10 +98,10 @@ void contiguous_container_storage_base::shrink_to_fit() noexcept set_size( mapping_, mapped_size() )().assume_succeeded(); } -void contiguous_container_storage_base::resize( std::size_t const target_size ) +void * contiguous_container_storage_base::resize( std::size_t const target_size ) { - if ( target_size > mapped_size() ) expand( target_size ); - else shrink( target_size ); + if ( target_size > mapped_size() ) return grow_to( target_size ); + else return shrink_to( target_size ); } void contiguous_container_storage_base::reserve( std::size_t const new_capacity ) diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 87daea8..11b5db5 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -221,7 +221,9 @@ TEST( bp_tree, playground ) EXPECT_EQ( bpt.find( -42 ), bpt.end() ); bpt.clear(); +# ifndef __APPLE__ // linker error (the workaround at EOF does not help) bpt.print(); +# endif } } @@ -265,3 +267,10 @@ TEST( bp_tree, nonunique ) //------------------------------------------------------------------------------ } // namespace psi::vm //------------------------------------------------------------------------------ +#ifdef __APPLE__ // Xcode 16.1 Symbol not found: __ZNSt3__119__is_posix_terminalEP7__sFILE +namespace std { inline namespace __1 { +#include +[[ gnu::weak, gnu::visibility( "default" ) ]] +extern bool __is_posix_terminal(FILE* __stream) { return isatty(fileno(__stream)); } +}} +#endif \ No newline at end of file