diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index 728c820..e71cc6f 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -104,4 +104,4 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: > cd ${{ steps.strings.outputs.build-output-dir }}/test && - ctest --build-config ${{ matrix.build_type }} + ctest --progress --output-on-failure --build-config ${{ matrix.build_type }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 274fb4e..2ce1327 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,16 +19,21 @@ include( ${CMAKE_CURRENT_BINARY_DIR}/cmake/get_cpm.cmake ) # Add packages -set( boost_ver boost-1.85.0 ) +set( boost_ver boost-1.86.0 ) CPMAddPackage( "gh:boostorg/static_assert#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/throw_exception#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/config#${boost_ver}" ) # Boost::core dependency +CPMAddPackage( "gh:boostorg/intrusive#${boost_ver}" ) # Boost::container dependency CPMAddPackage( "gh:boostorg/io#${boost_ver}" ) # Boost::utility dependency CPMAddPackage( "gh:boostorg/type_traits#${boost_ver}" ) # Boost::utility dependency CPMAddPackage( "gh:boostorg/predef#${boost_ver}" ) # Boost::winapi dependency CPMAddPackage( "gh:boostorg/assert#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/container#${boost_ver}" ) # used only for comparative benchmarking CPMAddPackage( "gh:boostorg/core#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/integer#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/move#${boost_ver}" ) CPMAddPackage( "gh:boostorg/preprocessor#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/stl_interfaces#${boost_ver}" ) CPMAddPackage( "gh:boostorg/winapi#${boost_ver}" ) CPMAddPackage( "gh:boostorg/utility#${boost_ver}" ) @@ -55,7 +60,9 @@ if ( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" ) PSI_add_link_options( Release -flto ) # lld does not seem to be enough add_compile_options( -stdlib=libc++ ) # Needed under WSL for some reason? - PSI_add_link_options( Debug -lc++ -lc++abi -lubsan ) + PSI_add_link_options( Debug -lc++ -lc++abi -lm -lubsan ) + PSI_add_link_options( DevRelease -lc++ -lc++abi -lm -lubsan ) + PSI_add_link_options( Release -lc++ -lc++abi -lm ) else() set( CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE true ) PSI_add_link_options( Release ${PSI_linker_LTO} ) @@ -69,9 +76,13 @@ endif() include( vm.cmake ) target_link_libraries( psi_vm PUBLIC + Boost::container Boost::core Boost::assert + Boost::integer + Boost::move Boost::preprocessor + Boost::stl_interfaces Boost::winapi Boost::utility ) @@ -83,6 +94,8 @@ target_include_directories( psi_vm PUBLIC "${std_fix_SOURCE_DIR}/include" # vm::vector uses it (in a header) ) +target_precompile_headers( psi_vm PUBLIC src/pch.hpp ) + ################### ## Testing diff --git a/README.md b/README.md index 5175693..3505e75 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,25 @@ -### vm - portable, lightweight, powerful, near-zero-overhead memory mapping and virtual memory management. +### vm - portable, lightweight, powerful, near-zero-overhead memory mapping, virtual memory management, containers and utilities. +(TODO: sync with https://github.com/ned14/llfio) -https://svn.boost.org/trac/boost/ticket/4827 -http://boost.2283326.n4.nabble.com/interprocess-shared-memory-lifetime-td2603982.html -http://lists.boost.org/Archives/boost/2010/10/172227.php -http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n2044.html +##### Sponsored by (at one point or another) -##### Submodule requirements - * config_ex - * err - * std_fix +https://farseer.com +https://microblink.com + + +##### Some ancient discussions + +https://svn.boost.org/trac/boost/ticket/4827 +http://boost.2283326.n4.nabble.com/interprocess-shared-memory-lifetime-td2603982.html +http://lists.boost.org/Archives/boost/2010/10/172227.php +http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n2044.html + +##### Dependencies + * Boost + * Psi.Build + * Psi.Err + * Psi.StdFix -##### C++ In-The-Kernel Now! ##### Copyright © 2011 - 2024. Domagoj Šarić. All rights reserved. diff --git a/include/psi/vm/align.hpp b/include/psi/vm/align.hpp index 2d185b2..ca372c8 100644 --- a/include/psi/vm/align.hpp +++ b/include/psi/vm/align.hpp @@ -18,7 +18,7 @@ namespace align_detail { [[ gnu::const ]] constexpr auto generic_divide_up( auto const numerator, auto const denominator ) noexcept { - return static_cast< decltype( numerator ) >( ( numerator + denominator - 1 ) / denominator ); + return static_cast( ( numerator + denominator - 1 ) / denominator ); } } // namespace detail @@ -36,10 +36,10 @@ namespace align_detail return __builtin_align_down( value, alignment ); else #endif - if constexpr ( std::is_pointer_v< T > ) - return std::bit_cast< T >( align_down( std::bit_cast< std::uintptr_t >( value ), alignment ) ); + if constexpr ( std::is_pointer_v ) + return std::bit_cast( align_down( std::bit_cast( value ), alignment ) ); else - return static_cast< T >( value / alignment * alignment ); + return static_cast( value / alignment * alignment ); } [[ using gnu: const, always_inline ]] constexpr auto align_up( auto const value, auto const alignment ) noexcept { @@ -49,21 +49,21 @@ namespace align_detail return __builtin_align_up( value, alignment ); else #endif - if constexpr ( std::is_pointer_v< T > ) - return std::bit_cast< T >( align_up( std::bit_cast< std::uintptr_t >( value ), alignment ) ); + if constexpr ( std::is_pointer_v ) + return std::bit_cast( align_up( std::bit_cast( value ), alignment ) ); else - return static_cast< T >( align_detail::generic_divide_up( value, alignment ) * alignment ); + return static_cast( align_detail::generic_divide_up( value, alignment ) * alignment ); } -template < unsigned alignment > [[ using gnu: const, always_inline ]] constexpr auto align_down( auto const value ) noexcept { return align_down( value, alignment ); } -template < unsigned alignment > [[ using gnu: const, always_inline ]] constexpr auto align_up ( auto const value ) noexcept { return align_up ( value, alignment ); } +template [[ using gnu: const, always_inline ]] constexpr auto align_down( auto const value ) noexcept { return align_down( value, alignment ); } +template [[ using gnu: const, always_inline ]] constexpr auto align_up ( auto const value ) noexcept { return align_up ( value, alignment ); } [[ using gnu: const, always_inline ]] constexpr auto divide_up( auto const numerator, auto const denominator ) noexcept { #ifdef __GNUC__ if ( __builtin_constant_p( denominator ) && /*is power of 2*/std::has_single_bit( unsigned( denominator ) ) ) - return static_cast< decltype( numerator ) >( align_up( numerator, denominator ) / denominator ); + return static_cast( align_up( numerator, denominator ) / denominator ); else #endif // GCC&co. return align_detail::generic_divide_up( numerator, denominator ); diff --git a/include/psi/vm/containers/abi.hpp b/include/psi/vm/containers/abi.hpp new file mode 100644 index 0000000..3bc3d29 --- /dev/null +++ b/include/psi/vm/containers/abi.hpp @@ -0,0 +1,107 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +PSI_WARNING_DISABLE_PUSH() +PSI_WARNING_MSVC_DISABLE( 5030 ) // unrecognized attribute + +//////////////////////////////////////////////////////////////////////////////// +// Modern(ized) attempt at 'automatized' boost::call_traits primarily to support +// efficient transparent comparators & non-inlined generic lookup functions +// which cause neither unnecessary copies of non-trivial types nor pass-by-ref +// of trivial ones. +// Largely still WiP... +// Essentially this is 'explicit IPA SROA'. +// https://gcc.gnu.org/onlinedocs/gccint/passes-and-files-of-the-compiler/inter-procedural-optimization-passes.html +//////////////////////////////////////////////////////////////////////////////// + +template +bool constexpr can_be_passed_in_reg +{ + ( + std::is_trivial_v && + ( sizeof( T ) <= 2 * sizeof( void * ) ) // assuming a sane ABI like SysV (ignoring the MS x64 disaster) + ) +#if defined( __GNUC__ ) || defined( __clang__ ) + || // detect SIMD types (this could also produce false positives for large compiler-native vectors that do not fit into the register file) + requires{ __builtin_convertvector( T{}, T ); } +#endif + // This is certainly not an exhaustive list/'trait' - certain types that can + // be passed in reg cannot be detected as such by existing compiler + // functionality, e.g. Homogeneous Vector Aggregates + // https://devblogs.microsoft.com/cppblog/introducing-vector-calling-convention + // users are encouraged to provide specializations for such types. +}; // can_be_passed_in_reg + +template +struct optimal_const_ref { using type = T const &; }; + +template +struct optimal_const_ref> { using type = std::basic_string_view; }; + +template +struct optimal_const_ref { using type = std::span const>; }; + +template +struct [[ clang::trivial_abi ]] pass_in_reg +{ + static auto constexpr pass_by_val{ can_be_passed_in_reg }; + + using value_type = T; + using stored_type = std::conditional_t::type>; + BOOST_FORCEINLINE + constexpr pass_in_reg( auto const &... args ) noexcept requires requires { stored_type{ args... }; } : value{ args... } {} + constexpr pass_in_reg( pass_in_reg const & ) noexcept = default; + constexpr pass_in_reg( pass_in_reg && ) noexcept = default; + + stored_type value; + + [[ gnu::pure ]] BOOST_FORCEINLINE + constexpr operator stored_type const &() const noexcept { return value; } +}; // pass_in_reg +template +pass_in_reg( T ) -> pass_in_reg; + +template +struct [[ clang::trivial_abi ]] pass_rv_in_reg +{ + static auto constexpr pass_by_val{ can_be_passed_in_reg }; + + using value_type = T; + using stored_type = std::conditional_t; + + constexpr pass_rv_in_reg( T && u ) noexcept : value{ std::move( u ) } {} // move for not-trivially-moveable yet trivial_abi types (that can be passed in reg) + + stored_type value; + + [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type const & () const & noexcept { return value ; } + [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type &&() && noexcept { return std::move( value ); } +}; // pass_rv_in_reg + +template bool constexpr reg { can_be_passed_in_reg }; +template bool constexpr reg>{ true }; +template bool constexpr reg>{ true }; + +template +concept Reg = reg; + +// 'Explicit IPA SROA' / pass-in-reg helper end +//////////////////////////////////////////////////////////////////////////////// + +PSI_WARNING_DISABLE_POP() + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp new file mode 100644 index 0000000..ba830e2 --- /dev/null +++ b/include/psi/vm/containers/b+tree.hpp @@ -0,0 +1,2294 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +PSI_WARNING_DISABLE_PUSH() +PSI_WARNING_MSVC_DISABLE( 5030 ) // unrecognized attribute + +namespace detail +{ + template + [[ gnu::const ]] auto header_data( std::span const hdr_storage ) noexcept + { + auto const data { align_up( hdr_storage.data() ) }; + auto const remaining_space{ hdr_storage.size() - unsigned( data - hdr_storage.data() ) }; + BOOST_ASSERT( remaining_space >= sizeof( Header ) ); + return std::pair{ reinterpret_cast
( data ), std::span{ data + sizeof( Header ), remaining_space - sizeof( Header ) } }; + } +} // namespace detail + +template +concept LookupType = transparent_comparator || std::is_same_v; + +template +concept InsertableType = ( transparent_comparator && std::is_convertible_v ) || std::is_same_v; + + +// user specializations are allowed and intended: + +template constexpr bool is_simple_comparator{ false }; +template constexpr bool is_simple_comparator>{ true }; +template constexpr bool is_simple_comparator>{ true }; + +template constexpr bool is_statically_sized { true }; +template requires requires{ T{}.size(); } constexpr bool is_statically_sized{ T{}.size() != 0 }; + +template +constexpr bool use_linear_search_for_sorted_array +{ + ( is_simple_comparator ) && + ( std::is_trivially_copyable_v ) && + ( sizeof( Key ) < ( 4 * sizeof( void * ) ) ) && + ( maximum_array_length * sizeof( Key ) <= 4096 ) && + ( is_statically_sized ) +}; // use_linear_search_for_sorted_array + + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base +//////////////////////////////////////////////////////////////////////////////// + +class bptree_base +{ +public: + using size_type = std::size_t; + using difference_type = std::make_signed_t; + using storage_result = err::fallible_result; + + [[ gnu::pure ]] bool empty() const noexcept { return BOOST_UNLIKELY( size() == 0 ); } + + void clear() noexcept; + + bptree_base( header_info hdr_info = {} ) noexcept; + + std::span user_header_data() noexcept; + + bool has_attached_storage() const noexcept { return nodes_.has_attached_storage(); } + + storage_result map_file( auto const file, flags::named_object_construction_policy const policy ) noexcept + { + storage_result success{ nodes_.map_file( file, policy ) }; +# ifndef NDEBUG + if ( std::move( success ) ) + { + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); + } +# endif + if ( std::move( success ) && nodes_.empty() ) + hdr() = {}; + return success; + } + storage_result map_memory( std::uint32_t initial_capacity_as_number_of_nodes = 0 ) noexcept; + +protected: +#if 0 // favoring CPU cache & branch prediction (linear scans) _vs_ TLB and disk access related issues, TODO make this configurable + static constexpr auto node_size{ page_size }; +#else + static constexpr auto node_size{ 256 }; +#endif + + using depth_t = std::uint8_t; + + template + // ceil( m / 2 ) + static constexpr auto ihalf_ceil{ static_cast( ( value + 1 ) / 2 ) }; + + struct [[ nodiscard, clang::trivial_abi ]] node_slot // instead of node pointers we store offsets - slots in the node pool + { + using value_type = std::uint32_t; + static node_slot const null; + value_type index{ static_cast( -1 ) }; // in-pool index/offset + [[ gnu::pure ]] value_type operator*() const noexcept { BOOST_ASSUME( index != null.index ); return index; } + [[ gnu::pure ]] bool operator==( node_slot const other ) const noexcept { return this->index == other.index; } + [[ gnu::pure ]] explicit operator bool() const noexcept { return index != null.index; } + }; // struct node_slot + + struct [[ nodiscard, clang::trivial_abi ]] node_header + { + static auto constexpr minimum_header_size{ 3 * sizeof( node_slot ) + 2 * sizeof( /*minimum size_type*/ std::uint8_t ) }; + using size_type = typename boost::uint_value_t::least; + + // At minimum we need a single-linked/directed list in the vertical/depth + // and horizontal/breadth directions (and the latter only for the leaf + // level - to have a connected sorted 'list' of all the values). + // However having a precise vertical back/up-link (parent_child_idx): + // * speeds up walks up the tree (as the parent (separator) key slots do + // not have to be searched for) + // * simplifies code (enabling several functions to become independent + // of the comparator - no longer need searching - and moved up into + // the base bptree classes) + // * while at the same time being a negligible overhead considering we + // are targeting much larger (page size sized) nodes. + node_slot parent {}; + node_slot left {}; + node_slot right {}; + size_type num_vals {}; + size_type parent_child_idx{}; + /* TODO + size_type start; // make keys and children arrays function as devectors: allow empty space at the beginning to avoid moves for smaller borrowings + */ + + [[ gnu::pure ]] bool is_root() const noexcept { return !parent; } + + protected: // merely to prevent slicing (in return-node-by-ref cases) + constexpr node_header( node_header const & ) noexcept = default; + constexpr node_header( node_header && ) noexcept = default; + public: + constexpr node_header( ) noexcept = default; + constexpr node_header & operator=( node_header && ) noexcept = default; + constexpr node_header & operator=( node_header const & ) noexcept = default; + }; // struct node_header + using node_size_type = node_header::size_type; + + struct alignas( node_size ) node_placeholder : node_header {}; + struct alignas( node_size ) free_node : node_header {}; + + // SCARY iterator parts + struct iter_pos + { + node_slot node {}; + node_size_type value_offset{}; + }; + + class base_iterator; + class base_random_access_iterator; + + struct insert_pos_t { node_slot node; node_size_type next_insert_offset; }; // TODO 'merge' with iter_pos + + struct find_pos0 // msvc pass-in-reg facepalm + { + node_size_type pos : ( sizeof( node_size_type ) * CHAR_BIT - 1 ); + node_size_type exact_find : 1; + }; + struct find_pos1 + { + node_size_type pos; + bool exact_find; + }; + + struct header // or persisted data members + { + node_slot root_; + node_slot first_leaf_; + node_slot last_leaf_; + node_slot free_list_; + node_slot::value_type free_node_count_{}; + size_t size_ {}; + depth_t depth_{}; + }; // struct header + + using node_pool = vm::vector; + +protected: + void swap( bptree_base & other ) noexcept; + + base_iterator make_iter( iter_pos ) noexcept; + base_iterator make_iter( node_slot, node_size_type offset ) noexcept; + base_iterator make_iter( node_header const &, node_size_type offset ) noexcept; + base_iterator make_iter( insert_pos_t ) noexcept; + + [[ gnu::pure ]] iter_pos begin_pos() const noexcept; + [[ gnu::pure ]] iter_pos end_pos() const noexcept; + + [[ gnu::pure ]] base_iterator begin() noexcept; + [[ gnu::pure ]] base_iterator end () noexcept; + + [[ gnu::pure ]] base_random_access_iterator ra_begin() noexcept; + [[ gnu::pure ]] base_random_access_iterator ra_end () noexcept; + + [[ gnu::pure ]] size_type size() const noexcept { return hdr().size_; } + + [[ gnu::cold ]] node_header & create_root(); + + [[ gnu::pure ]] static bool underflowed( auto const & node ) noexcept { return node.num_vals < node.min_values; } + [[ gnu::pure ]] static bool can_borrow ( auto const & node ) noexcept { return node.num_vals > node.min_values; } + + [[ gnu::pure ]] depth_t leaf_level( ) const noexcept; + [[ gnu::pure ]] bool is_leaf_level( depth_t const level ) const noexcept; + + void free ( node_header & ) noexcept; + void free_leaf( node_header & ) noexcept; + + void reserve_additional( node_slot::value_type additional_nodes ); + void reserve ( node_slot::value_type new_capacity_in_number_of_nodes ); + + [[ gnu::pure ]] header & hdr() noexcept; + [[ gnu::pure ]] header const & hdr() const noexcept { return const_cast( *this ).hdr(); } + + node_slot first_leaf() const noexcept { return hdr().first_leaf_; } + + static void verify( auto const & node ) noexcept + { + BOOST_ASSUME( node.num_vals <= node.max_values ); + // also used for underflowing nodes and (most problematically) for root nodes 'interpreted' as inner nodes...TODO... + //BOOST_ASSUME( node.num_vals >= node.min_values ); + } + + static constexpr auto keys ( auto & node ) noexcept { verify( node ); return std::span{ node.keys , static_cast( node.num_vals ) }; } + static constexpr auto keys ( auto const & node ) noexcept { verify( node ); return std::span{ node.keys , static_cast( node.num_vals ) }; } + static constexpr auto children( auto & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, static_cast( node.num_vals + 1U ) }; else return std::array{}; } + static constexpr auto children( auto const & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, static_cast( node.num_vals + 1U ) }; else return std::array{}; } + + [[ gnu::pure ]] static constexpr node_size_type num_vals ( auto const & node ) noexcept { return node.num_vals; } + [[ gnu::pure ]] static constexpr node_size_type num_chldrn( auto const & node ) noexcept { if constexpr ( requires{ node.children; } ) { BOOST_ASSUME( node.num_vals ); return node.num_vals + 1U; } else return 0; } + + template + static constexpr node_size_type size( auto const & node ) noexcept + { + if constexpr ( requires{ &(node.*array) == &node.keys; } ) return num_vals ( node ); + else return num_chldrn( node ); + } + + template + static auto rshift( auto & node, node_size_type const start_offset, node_size_type const end_offset ) noexcept + { + auto const max{ std::size( node.*array ) }; + BOOST_ASSUME( end_offset <= max ); + BOOST_ASSUME( start_offset < max ); + BOOST_ASSUME( start_offset < end_offset ); + auto const begin{ &(node.*array)[ start_offset ] }; + auto const end { &(node.*array)[ end_offset ] }; + auto const new_begin{ std::shift_right( begin, end, 1 ) }; + BOOST_ASSUME( new_begin == begin + 1 ); + return std::span{ new_begin, end }; + } + template static auto rshift( auto & node, node_size_type const offset ) noexcept { return rshift( node, offset, size( node ) ); } + template static auto rshift( auto & node ) noexcept { return rshift( node, 0 ); } + template + static auto lshift( auto & node, node_size_type const start_offset, node_size_type const end_offset ) noexcept + { + auto const max{ std::size( node.*array ) }; + BOOST_ASSUME( end_offset <= max ); + BOOST_ASSUME( start_offset < max ); + BOOST_ASSUME( start_offset < end_offset ); + auto const begin{ &(node.*array)[ start_offset ] }; + auto const end { &(node.*array)[ end_offset ] }; + auto const new_end{ std::shift_left( begin, end, 1 ) }; + BOOST_ASSUME( new_end == end - 1 ); + return std::span{ begin, new_end }; + } + template static auto lshift( auto & node, node_size_type const offset ) noexcept { return lshift( node, offset, size( node ) ); } + template static auto lshift( auto & node ) noexcept { return lshift( node, 0 ); } + + template static void rshift_keys( N & node, auto... args ) noexcept { rshift<&N::keys>( node, args... ); } + template static void lshift_keys( N & node, auto... args ) noexcept { lshift<&N::keys>( node, args... ); } + + template + void rshift_chldrn( N & parent, auto... args ) noexcept { + auto const shifted_children{ rshift<&N::children>( parent, static_cast( args )... ) }; + for ( auto ch_slot : shifted_children ) + node( ch_slot ).parent_child_idx++; + } + template + void lshift_chldrn( N & parent, auto... args ) noexcept { + auto const shifted_children{ lshift<&N::children>( parent, static_cast( args )... ) }; + for ( auto ch_slot : shifted_children ) + node( ch_slot ).parent_child_idx--; + } + + void rshift_sibling_parent_pos( node_header & node ) noexcept; + void update_right_sibling_link( node_header const & left_node, node_slot left_node_slot ) noexcept; + void unlink_and_free_node( node_header & node, node_header & cached_left_sibling ) noexcept; + void unlink_and_free_leaf( node_header & leaf, node_header & cached_left_sibling ) noexcept; + + void unlink_left ( node_header & nd ) noexcept; + void unlink_right( node_header & nd ) noexcept; + void link( node_header & left, node_header & right ) const noexcept; + + [[ gnu::sysv_abi ]] + std::pair new_spillover_node_for( node_header & existing_node ); + + node_placeholder & new_root( node_slot left_child, node_slot right_child ); + + template + static NodeType & as( SourceNode & slot ) noexcept + { + static_assert( sizeof( NodeType ) == sizeof( slot ) || std::is_same_v ); + return static_cast( static_cast( slot ) ); + } + template + static NodeType const & as( SourceNode const & slot ) noexcept { return as( const_cast( slot ) ); } + + [[ gnu::pure ]] node_placeholder & node( node_slot const offset ) noexcept { return nodes_[ *offset ]; } + [[ gnu::pure ]] node_placeholder const & node( node_slot const offset ) const noexcept { return nodes_[ *offset ]; } + + auto & root() noexcept { return node( hdr().root_ ); } + auto const & root() const noexcept { return const_cast( *this ).root(); } + + template N & node( node_slot const offset ) noexcept { return as( node( offset ) ); } + template N const & node( node_slot const offset ) const noexcept { return as( node( offset ) ); } + + template N & right( N const & nd ) noexcept { return node( nd.right ); } + template N const & right( N const & nd ) const noexcept { return node( nd.right ); } + template N & left ( N const & nd ) noexcept { return node( nd.left ); } + template N const & left ( N const & nd ) const noexcept { return node( nd.left ); } + + [[ gnu::pure ]] bool is_my_node( node_header const & ) const noexcept; + [[ gnu::pure ]] node_slot slot_of ( node_header const & ) const noexcept; + + static bool full( auto const & node ) noexcept { + BOOST_ASSUME( node.num_vals <= node.max_values ); + return node.num_vals == node.max_values; + } + + [[ nodiscard ]] node_placeholder & new_node(); + + template + [[ nodiscard ]] N & new_node() { return as( new_node() ); } + +private: + auto header_data() noexcept { return detail::header_data
( nodes_.user_header_data() ); } + + void assign_nodes_to_free_pool( node_slot::value_type starting_node ) noexcept; + + void update_leaf_list_ends( node_header & removed_leaf ) noexcept; + +protected: + node_pool nodes_; +#ifndef NDEBUG // debugging helpers (undoing type erasure done by contiguous_container_storage_base) + header const * p_hdr_ {}; + node_placeholder const * p_nodes_{}; +#endif +}; // class bptree_base + +inline constexpr bptree_base::node_slot const bptree_base::node_slot::null{ static_cast( -1 ) }; + + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base::base_iterator +//////////////////////////////////////////////////////////////////////////////// + +class bptree_base::base_iterator +{ +public: + constexpr base_iterator() noexcept = default; + + base_iterator & operator++() noexcept; + base_iterator & operator--() noexcept; + + bool operator==( base_iterator const & ) const noexcept; + +public: // extensions + iter_pos const & pos() const noexcept { return pos_; } + +protected: + friend class bptree_base; + + base_iterator( node_pool &, iter_pos ) noexcept; + + [[ gnu::pure ]] node_header & node() const noexcept; + + mutable +#ifndef NDEBUG // for bounds checking + std::span +#else + node_placeholder * __restrict +#endif + nodes_{}; + iter_pos pos_ {}; + +private: + template + friend class bp_tree; + void update_pool_ptr( node_pool & ) const noexcept; +}; // class base_iterator + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base::base_random_access_iterator +//////////////////////////////////////////////////////////////////////////////// + +class bptree_base::base_random_access_iterator : public base_iterator +{ +protected: + size_type index_; + + friend class bptree_base; + template friend class bp_tree; + + base_random_access_iterator( bptree_base & parent, iter_pos const pos, size_type const start_index ) noexcept + : base_iterator{ parent.nodes_, pos }, index_{ start_index } {} + +public: + constexpr base_random_access_iterator() noexcept = default; + + [[ clang::no_sanitize( "unsigned-integer-overflow" ) ]] + difference_type operator-( base_random_access_iterator const & other ) const noexcept { return static_cast( this->index_ - other.index_ ); } + + base_random_access_iterator & operator+=( difference_type n ) noexcept; + + base_random_access_iterator & operator++( ) noexcept { base_iterator::operator++(); ++index_; return *this; } + base_random_access_iterator operator++(int) noexcept { auto current{ *this }; operator++(); return current; } + base_random_access_iterator & operator--( ) noexcept { base_iterator::operator--(); --index_; return *this; } + base_random_access_iterator operator--(int) noexcept { auto current{ *this }; operator--(); return current; } + + // should implicitly handle end iterator comparison also (this requires the start_index constructor argument for the construction of end iterators) + friend constexpr bool operator==( base_random_access_iterator const & left, base_random_access_iterator const & right ) noexcept { return left.index_ == right.index_; } + + auto operator<=>( base_random_access_iterator const & other ) const noexcept { return this->index_ <=> other.index_; } +}; // class base_random_access_iterator + + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base_wkey +//////////////////////////////////////////////////////////////////////////////// + +template +class bptree_base_wkey : public bptree_base +{ +private: + template + using iter_impl = boost::stl_interfaces::iterator_interface + < +# if !BOOST_STL_INTERFACES_USE_DEDUCED_THIS + Impl, +# endif + Tag, + Key + >; + +public: + using key_type = Key; + using value_type = key_type; // TODO map support + + // support for non trivial types (for which move and pass-by-ref matters) is WiP, nowhere near complete + static_assert( std::is_trivial_v ); + + using key_rv_arg = std::conditional_t, Key const, pass_rv_in_reg>; + using key_const_arg = std::conditional_t, Key const, pass_in_reg >; + + class fwd_iterator; + class ra_iterator; + + using iterator = fwd_iterator; + using const_iterator = std::basic_const_iterator; + +public: + storage_result map_memory( size_type initial_capacity = 0 ) noexcept { return bptree_base::map_memory( node_count_required_for_values( initial_capacity ) ); } + size_type capacity() const noexcept + { + // TODO WiP playground + auto const n{ nodes_.capacity() }; + if ( !n ) [[ unlikely ]] + return 0; + + node_slot::value_type inner_nodes{ 0 }; + node_slot::value_type current_level_count{ 1 }; + while ( ( current_level_count * inner_node::max_children ) < ( n - inner_nodes ) ) + { + inner_nodes += current_level_count; + current_level_count *= inner_node::max_children; + } + + std::uint8_t const depth{ hdr().depth_ }; + std::uint8_t max_inner_node_count{ depth > 1 }; + for ( auto d{ 3 }; d < depth; ++d ) + { + max_inner_node_count += static_cast( max_inner_node_count * inner_node::max_children ); + } + BOOST_ASSUME( max_inner_node_count < n ); + return ( n - max_inner_node_count ) * leaf_node::max_values; + } + + void reserve_additional( size_type const additional_values ) { bptree_base::reserve_additional( node_count_required_for_values( additional_values ) ); } + void reserve ( size_type const new_capacity ) { bptree_base::reserve ( node_count_required_for_values( new_capacity ) ); } + + iterator erase( const_iterator iter ) noexcept; + + // solely a debugging helper (include b+tree_print.hpp) + void print() const; + +protected: // node types + struct alignas( node_size ) parent_node : node_header + { + static auto constexpr storage_space{ node_size - align_up( sizeof( node_header ), alignof( Key ) ) }; + + // storage_space = ( order - 1 ) * sizeof( key ) + order * sizeof( child_ptr ) + // storage_space = order * szK - szK + order * szC + // storage_space + szK = order * ( szK + szC ) + // order = ( storage_space + szK ) / ( szK + szC ) + static constexpr node_size_type order // "m" + { + ( storage_space + sizeof( Key ) ) + / + ( sizeof( Key ) + sizeof( node_slot ) ) + }; + + using value_type = Key; + + static node_size_type constexpr max_children{ order }; + static node_size_type constexpr max_values { max_children - 1 }; + + Key keys [ max_values ]; + node_slot children[ max_children ]; + }; // struct parent_node + + struct inner_node : parent_node + { + static node_size_type constexpr min_children{ ihalf_ceil }; + static node_size_type constexpr min_values { min_children - 1 }; + + static_assert( min_children >= 3 ); + }; // struct inner_node + + struct root_node : parent_node + { + static auto constexpr min_children{ 2 }; + static auto constexpr min_values { min_children - 1 }; + }; // struct root_node + + struct alignas( node_size ) leaf_node : node_header + { + // TODO support for maps (i.e. keys+values) + using value_type = Key; + + static node_size_type constexpr storage_space{ node_size - align_up( sizeof( node_header ), alignof( Key ) ) }; + static node_size_type constexpr max_values { storage_space / sizeof( Key ) }; + static node_size_type constexpr min_values { ihalf_ceil }; + + Key keys[ max_values ]; + }; // struct leaf_node + + static_assert( sizeof( inner_node ) == node_size ); + static_assert( sizeof( leaf_node ) == node_size ); + +protected: // split_to_insert and its helpers + root_node & new_root( node_slot const left_child, node_slot const right_child, key_rv_arg separator_key ) + { + auto & new_root_node{ as( bptree_base::new_root( left_child, right_child ) ) }; + new_root_node.keys [ 0 ] = std::move( separator_key ); + new_root_node.children[ 0 ] = left_child; + new_root_node.children[ 1 ] = right_child; + return new_root_node; + } + + auto insert_into_new_node + ( + inner_node & node, inner_node & new_node, + key_rv_arg value, + node_size_type const insert_pos, node_size_type const new_insert_pos, + node_slot const key_right_child + ) noexcept + { + BOOST_ASSUME( bool( key_right_child ) ); + + using N = inner_node; + + auto const max{ N::max_values }; + auto const mid{ N::min_values }; + + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + + // old node gets the minimum/median/mid -> the new gets the leftovers (i.e. mid or mid + 1) + new_node.num_vals = max - mid; + + value_type key_to_propagate; + if ( new_insert_pos == 0 ) + { + key_to_propagate = std::move( value ); + + move_keys ( node, mid , node.num_vals , new_node, 0 ); + move_chldrn( node, mid + 1, node.num_vals + 1, new_node, 1 ); + } + else + { + key_to_propagate = std::move( node.keys[ mid ] ); + + move_keys ( node, mid + 1, insert_pos , new_node, 0 ); + move_chldrn( node, mid + 1, insert_pos + 1, new_node, 0 ); + + move_keys ( node, insert_pos , node.num_vals , new_node, new_insert_pos ); + move_chldrn( node, insert_pos + 1, node.num_vals + 1, new_node, new_insert_pos + 1 ); + + keys( new_node )[ new_insert_pos - 1 ] = std::move( value ); + } + insrt_child( new_node, new_insert_pos, key_right_child ); + + node.num_vals = mid; + + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + + return std::make_pair( key_to_propagate, new_insert_pos ); + } + + static auto insert_into_new_node + ( + leaf_node & node, leaf_node & new_node, + key_rv_arg value, + node_size_type const insert_pos, node_size_type const new_insert_pos, + node_slot const key_right_child + ) noexcept + { + BOOST_ASSUME( !key_right_child ); + + using N = leaf_node; + + auto const max{ N::max_values }; + auto const mid{ N::min_values }; + + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + + move_keys( node, mid , insert_pos, new_node, 0 ); + move_keys( node, insert_pos, max , new_node, new_insert_pos + 1 ); + + node .num_vals = mid ; + new_node.num_vals = max - mid + 1; + + keys( new_node )[ new_insert_pos ] = std::move( value ); + auto const & key_to_propagate{ new_node.keys[ 0 ] }; + + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + + return std::make_pair( key_to_propagate, static_cast( new_insert_pos + 1 ) ); + } + + auto insert_into_existing_node( inner_node & node, inner_node & new_node, key_rv_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + { + BOOST_ASSUME( bool( key_right_child ) ); + + using N = inner_node; + + auto const max{ N::max_values }; + auto const mid{ N::min_values }; + + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + + value_type key_to_propagate{ std::move( node.keys[ mid - 1 ] ) }; + + move_keys ( node, mid, num_vals ( node ), new_node, 0 ); + move_chldrn( node, mid, num_chldrn( node ), new_node, 0 ); + + rshift_keys ( node, insert_pos , mid ); + rshift_chldrn( node, insert_pos + 1, mid + 1 ); + + node .num_vals = mid; + new_node.num_vals = max - mid; + + keys ( node )[ insert_pos ] = std::move( value ); + insrt_child( node, insert_pos + 1, key_right_child ); + + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + + return std::make_pair( std::move( key_to_propagate ), static_cast( insert_pos + 1 ) ); + } + + static auto insert_into_existing_node( leaf_node & node, leaf_node & new_node, key_rv_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + { + BOOST_ASSUME( !key_right_child ); + + using N = leaf_node; + + auto const max{ N::max_values }; + auto const mid{ N::min_values }; + + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + + move_keys( node, mid - 1 , max, new_node, 0 ); + rshift_keys( node, insert_pos, mid ); + + node .num_vals = mid; + new_node.num_vals = max - mid + 1; + + keys( node )[ insert_pos ] = std::move( value ); + auto const & key_to_propagate{ new_node.keys[ 0 ] }; + + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + + return std::make_pair( key_to_propagate, static_cast( insert_pos + 1 ) ); + } + + template + insert_pos_t split_to_insert( N & node_to_split, node_size_type const insert_pos, key_rv_arg value, node_slot const key_right_child ) + { + auto const max{ N::max_values }; + auto const mid{ N::min_values }; + BOOST_ASSUME( node_to_split.num_vals == max ); + auto [node_slot, new_slot]{ bptree_base::new_spillover_node_for( node_to_split ) }; + auto p_node { &node( node_slot ) }; + auto p_new_node{ &node( new_slot ) }; + verify( *p_node ); + BOOST_ASSUME( p_node->num_vals == max ); + BOOST_ASSERT + ( + !p_node->parent || ( node( p_node->parent ).children[ p_node->parent_child_idx ] == node_slot ) + ); + + auto const new_insert_pos { insert_pos - mid }; + bool const insertion_into_new_node{ new_insert_pos >= 0 }; + auto [key_to_propagate, next_insert_pos]{ insertion_into_new_node // we cannot save a reference here because it might get invalidated by the new_node() call below + ? insert_into_new_node ( *p_node, *p_new_node, std::move( value ), insert_pos, static_cast( new_insert_pos ), key_right_child ) + : insert_into_existing_node( *p_node, *p_new_node, std::move( value ), insert_pos, key_right_child ) + }; + + verify( *p_node ); + verify( *p_new_node ); + BOOST_ASSUME( p_node->num_vals == mid ); + + if ( std::is_same_v && !p_new_node->right ) { + hdr().last_leaf_ = new_slot; + } + + // propagate the mid key to the parent + if ( p_node->is_root() ) [[ unlikely ]] { + new_root( node_slot, new_slot, std::move( key_to_propagate ) ); + } else { + auto const key_pos{ static_cast( p_new_node->parent_child_idx /*it is the _right_ child*/ - 1 ) }; + insert( node( p_node->parent ), key_pos, std::move( key_to_propagate ), new_slot ); + } + return insertion_into_new_node + ? insert_pos_t{ new_slot, next_insert_pos } + : insert_pos_t{ node_slot, next_insert_pos }; + } + + +protected: // 'other' + // key_locations (containing find_pos) should also be returnable through registers + using find_pos = std::conditional_t + < + ( sizeof( find_pos1 ) > 2 ) && + ( leaf_node::max_values <= ( std::numeric_limits::max() / 2 ) ), // we get only half the range if one bit is shaved off for exact_find + bptree_base::find_pos0, + bptree_base::find_pos1 + >; + struct key_locations + { + leaf_node & leaf; + find_pos leaf_offset; + // optional - if also present in an inner node as a separator key + node_size_type inner_offset; // ordered for compact layout + node_slot inner; + }; // struct key_locations + + [[ gnu::pure, nodiscard ]] iterator make_iter( auto const &... args ) noexcept { return static_cast( bptree_base::make_iter( args... ) ); } + [[ gnu::pure, nodiscard ]] iterator make_iter( key_locations const loc ) noexcept { return make_iter( loc.leaf, loc.leaf_offset.pos ); } + + template + insert_pos_t insert( N & target_node, node_size_type const target_node_pos, key_rv_arg v, node_slot const right_child ) + { + verify( target_node ); + if ( full( target_node ) ) [[ unlikely ]] { + return split_to_insert( target_node, target_node_pos, std::move( v ), right_child ); + } else { + ++target_node.num_vals; + rshift_keys( target_node, target_node_pos ); + target_node.keys[ target_node_pos ] = std::move( v ); + if constexpr ( requires { target_node.children; } ) { + node_size_type const ch_pos( target_node_pos + /*>right< child*/ 1 ); + rshift_chldrn( target_node, ch_pos ); + this->insrt_child( target_node, ch_pos, right_child ); + } + return { slot_of( target_node ), static_cast( target_node_pos + 1 ) }; + } + } + + iter_pos erase( leaf_node & leaf, node_size_type const leaf_key_offset ) noexcept + { + auto & hdr { this->hdr() }; + auto & depth_{ hdr.depth_ }; + + iter_pos next_pos{ slot_of( leaf ), leaf_key_offset }; + + lshift_keys( leaf, leaf_key_offset ); + --leaf.num_vals; + + if ( depth_ == 1 ) [[ unlikely ]] // handle 'leaf root' deletion directly to simplify handle_underflow() + { + auto & root_{ hdr.root_ }; + BOOST_ASSUME( root_ == slot_of( leaf ) ); + BOOST_ASSUME( leaf.is_root() ); + BOOST_ASSUME( hdr.size_ == leaf.num_vals + 1 ); + BOOST_ASSUME( !leaf.left ); + BOOST_ASSUME( !leaf.right ); + if ( leaf.num_vals == 0 ) + { + root_ = {}; + free( leaf ); + --depth_; + BOOST_ASSUME( depth_ == 0 ); + BOOST_ASSUME( hdr.size_ == 1 ); + BOOST_ASSUME( !hdr.first_leaf_ ); + BOOST_ASSUME( !hdr.last_leaf_ ); + next_pos = {}; // empty end_pos + } + } + else + { + auto p_leaf{ &leaf }; + bool last_node_value_was_erased{ leaf_key_offset == p_leaf->num_vals }; + if ( underflowed( leaf ) ) + { + BOOST_ASSUME( !leaf.is_root() ); + BOOST_ASSUME( depth_ > 1 ); + next_pos = handle_underflow( leaf, leaf_level() ); + next_pos.value_offset += leaf_key_offset; + p_leaf = &this->leaf( next_pos.node ); + BOOST_ASSUME( next_pos.value_offset <= p_leaf->num_vals ); + last_node_value_was_erased = ( next_pos.value_offset == p_leaf->num_vals ); + } + + if ( last_node_value_was_erased ) + { + if ( !p_leaf->right ) { + next_pos = bptree_base::end_pos(); + } else { + next_pos.node = p_leaf->right; + next_pos.value_offset = 0; + } + } + } + + --hdr.size_; + return next_pos; + } + + // This function only serves the purpose of maintaining the rule about the + // minimum number of children per node - that rule is actually only + // 'academic' (for making sure that the performance/complexity guarantees + // will be maintained) - the tree would operate correctly even without + // maintaining that invariant (TODO make this an option). + bool bulk_append_fill_incomplete_leaf( leaf_node & leaf ) noexcept + { + auto const missing_keys{ static_cast( std::max( 0, signed( leaf.min_values - leaf.num_vals ) ) ) }; + if ( missing_keys ) + { + auto & preceding{ left( leaf ) }; + BOOST_ASSUME( preceding.num_vals + leaf.num_vals >= leaf_node::min_values * 2 ); + std::shift_right( &leaf.keys[ 0 ], &leaf.keys[ leaf.num_vals + missing_keys ], missing_keys ); + this->move_keys( preceding, preceding.num_vals - missing_keys, preceding.num_vals, leaf, 0 ); + leaf .num_vals += missing_keys; + preceding.num_vals -= missing_keys; + return true; + } + return false; + } + + void bulk_append( leaf_node * src_leaf, insert_pos_t rightmost_parent_pos ) + { + for ( ;; ) + { + BOOST_ASSUME( !src_leaf->parent ); + auto & rightmost_parent{ inner( rightmost_parent_pos.node ) }; + BOOST_ASSUME( rightmost_parent_pos.next_insert_offset == rightmost_parent.num_vals ); + auto const next_src_slot{ src_leaf->right }; + rightmost_parent_pos = insert + ( + rightmost_parent, + rightmost_parent_pos.next_insert_offset, + key_rv_arg{ /*mrmlj*/Key{ src_leaf->keys[ 0 ] } }, + slot_of( *src_leaf ) + ); + if ( !next_src_slot ) + break; + src_leaf = &leaf( next_src_slot ); + } + hdr().last_leaf_ = slot_of( *src_leaf ); + if ( bulk_append_fill_incomplete_leaf( *src_leaf ) ) + { + // Borrowing from the left sibling happened _after_ src_leaf was + // already inserted into the parent so we have to update the + // separator key in the parent (since this is the rightmost leaf we + // know that the separator key has to be in the immediate parent - + // no need to call the generic update_separator()). + auto & prnt{ parent( *src_leaf ) }; + BOOST_ASSUME( src_leaf->parent_child_idx == num_chldrn( prnt ) - 1 ); + keys( prnt ).back() = src_leaf->keys[ 0 ]; + } + } + + struct bulk_copied_input + { + node_slot begin; + iter_pos end; + size_type size; + }; + template + bulk_copied_input + bulk_insert_prepare( std::ranges::subrange keys ) + { + if ( keys.empty() ) [[ unlikely ]] + return bulk_copied_input{}; + + auto constexpr can_preallocate{ kind == std::ranges::subrange_kind::sized }; + if constexpr ( can_preallocate ) + reserve_additional( static_cast( keys.size() ) ); + else + reserve_additional( 42 ); + // w/o preallocation a saved hdr reference could get invalidated + auto const begin { can_preallocate ? hdr().free_list_ : slot_of( new_node() ) }; + auto leaf_slot{ begin }; + auto p_keys{ keys.begin() }; + size_type count{ 0 }; + for ( ;; ) + { + auto & leaf{ this->leaf( leaf_slot ) }; + BOOST_ASSUME( leaf.num_vals == 0 ); + if constexpr ( can_preallocate ) { + auto const size_to_copy{ static_cast( std::min( leaf.max_values, static_cast( keys.end() - p_keys ) ) ) }; + BOOST_ASSUME( size_to_copy ); + std::copy_n( p_keys, size_to_copy, leaf.keys ); + leaf.num_vals = size_to_copy; + count += size_to_copy; + p_keys += size_to_copy; + } else { + while ( ( p_keys != keys.end() ) && ( leaf.num_vals < leaf.max_values ) ) { + leaf.keys[ leaf.num_vals++ ] = *p_keys++; + } + count += leaf.num_vals; + } + BOOST_ASSUME( leaf.num_vals > 0 ); + --this->hdr().free_node_count_; + if ( p_keys != keys.end() ) + { + if constexpr ( can_preallocate ) { + leaf_slot = leaf.right; + } else { + auto & new_leaf{ new_node() }; + link( leaf, new_leaf ); + leaf_slot = slot_of( new_leaf ); + } + BOOST_ASSUME( !!leaf_slot ); + } + else + { + if constexpr ( can_preallocate ) { + this->hdr().free_list_ = leaf.right; + unlink_right( leaf ); + BOOST_ASSERT( count == static_cast( keys.size() ) ); + count = static_cast( keys.size() ); // eliminate the accumulation code above + } + return bulk_copied_input{ begin, { leaf_slot, leaf.num_vals }, count }; + } + } + std::unreachable(); + } + + void bulk_insert_into_empty( node_slot const begin_leaf, iter_pos const end_leaf, size_type const total_size ) + { + BOOST_ASSUME( empty() ); + auto * hdr{ &this->hdr() }; + hdr->root_ = begin_leaf; + hdr->first_leaf_ = begin_leaf; + BOOST_ASSUME( hdr->depth_ == 0 ); + if ( begin_leaf == end_leaf.node ) [[ unlikely ]] // single-node-sized initial insert + { + hdr->last_leaf_ = end_leaf.node; + hdr->size_ = total_size; + hdr->depth_ = ( total_size != 0 ); + return; + } + auto const & first_root_left { leaf ( begin_leaf ) }; + auto & first_root_right{ right( first_root_left ) }; + first_root_right.parent_child_idx = 1; + hdr->depth_ = 1; + auto const first_unconnected_node{ first_root_right.right }; + new_root( begin_leaf, first_root_left.right, key_rv_arg{ /*mrmlj*/Key{ first_root_right.keys[ 0 ] } } ); // may invalidate references + hdr = &this->hdr(); + BOOST_ASSUME( hdr->depth_ == 2 ); + if ( first_unconnected_node ) { // first check if there are more than two nodes + bulk_append( &leaf( first_unconnected_node ), { hdr->root_, 1 } ); + BOOST_ASSUME( hdr->last_leaf_ == end_leaf.node ); + } else { + hdr->last_leaf_ = end_leaf.node; + BOOST_ASSUME( !!hdr->last_leaf_ ); + } + hdr->size_ = total_size; + } + + [[ gnu::pure ]] leaf_node & leaf ( node_slot const slot ) noexcept { return node< leaf_node>( slot ); } + [[ gnu::pure ]] inner_node & inner ( node_slot const slot ) noexcept { return node( slot ); } + [[ gnu::pure ]] inner_node & parent( node_header & child ) noexcept { return inner( child.parent ); } + + leaf_node const & leaf ( node_slot const slot ) const noexcept { return const_cast( *this ).leaf( slot ); } + inner_node const & parent( node_header const & child ) const noexcept { return const_cast( *this ).parent( const_cast( child ) ); } + + void update_separator( leaf_node & leaf, Key const & new_separator ) noexcept + { + // the leftmost leaf does not have a separator key (at all) + if ( !leaf.left ) [[ unlikely ]] + { + BOOST_ASSUME( leaf.parent_child_idx == 0 ); + BOOST_ASSUME( hdr().first_leaf_ == slot_of( leaf ) ); + return; + } + auto const & separator_key{ leaf.keys[ 0 ] }; + BOOST_ASSUME( separator_key != new_separator ); + // a leftmost child does not have a key (in the immediate parent) + // (because a left child is strictly less-than its separator key - for + // the leftmost child there is no key further left that could be + // greater-or-equal to it) + auto parent_child_idx{ leaf.parent_child_idx }; + auto * parent { &this->parent( leaf ) }; + while ( parent_child_idx == 0 ) + { + parent = &this->parent( *parent ); + parent_child_idx = parent->parent_child_idx; + } + // can be zero only for the leftmost leaf which was checked for in the + // loop above and at the beginning of the function + BOOST_ASSUME( parent_child_idx > 0 ); + auto & parent_key{ parent->keys[ leaf.parent_child_idx - 1 ] }; + BOOST_ASSUME( parent_key == separator_key ); + parent_key = new_separator; + } + + template + BOOST_NOINLINE + iter_pos handle_underflow( N & node, depth_t const level ) noexcept + { + BOOST_ASSUME( underflowed( node ) ); + + auto constexpr parent_node_type{ requires{ node.children; } }; + auto constexpr leaf_node_type { !parent_node_type }; + + BOOST_ASSUME( level > 0 ); + BOOST_ASSUME( !leaf_node_type || level == leaf_level() ); + auto const node_slot{ slot_of( node ) }; + auto & parent{ this->node( node.parent ) }; + verify( parent ); + + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); + auto const parent_child_idx { node.parent_child_idx }; + bool const parent_has_key_copy{ leaf_node_type && ( parent_child_idx > 0 ) }; + auto const parent_key_idx { parent_child_idx - parent_has_key_copy }; + BOOST_ASSUME( !parent_has_key_copy || parent.keys[ parent_key_idx ] == node.keys[ 0 ] ); + + BOOST_ASSUME( parent.children[ parent_child_idx ] == node_slot ); + // the left and right level dlink pointers can point 'across' parents + // (and so cannot be used to resolve existence of siblings) + auto const has_right_sibling{ parent_child_idx < ( num_chldrn( parent ) - 1 ) }; + auto const has_left_sibling { parent_child_idx > 0 }; + auto const p_right_sibling { has_right_sibling ? &right( node ) : nullptr }; + auto const p_left_sibling { has_left_sibling ? &left ( node ) : nullptr }; + + auto const right_separator_key_idx{ static_cast( parent_key_idx + parent_has_key_copy ) }; + auto const left_separator_key_idx{ std::min( static_cast( right_separator_key_idx - 1 ), parent.num_vals ) }; // (ab)use unsigned wraparound + auto const p_right_separator_key { has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; + auto const p_left_separator_key { has_left_sibling ? &parent.keys[ left_separator_key_idx ] : nullptr }; + + // save&return the node (and offset) that the underflowed node's values + // end up + auto final_node { node_slot }; + node_size_type final_node_original_keys_offset{ 0 }; + + BOOST_ASSUME( has_right_sibling || has_left_sibling ); + BOOST_ASSERT( &node != p_left_sibling ); + BOOST_ASSERT( &node != p_right_sibling ); + BOOST_ASSERT( static_cast( &node ) != static_cast( &parent ) ); + // Borrow from left sibling if possible + if ( p_left_sibling && can_borrow( *p_left_sibling ) ) + { + verify( *p_left_sibling ); + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); + node.num_vals++; + rshift_keys( node ); + BOOST_ASSUME( node.num_vals == node.min_values ); + auto const node_keys{ keys( node ) }; + auto const left_keys{ keys( *p_left_sibling ) }; + + if constexpr ( leaf_node_type ) { + // Move the largest key from left sibling to the current node + BOOST_ASSUME( parent_has_key_copy ); + node_keys.front() = std::move( left_keys.back() ); + // adjust the separator key in the parent + BOOST_ASSERT( *p_left_separator_key == node_keys[ 1 ] ); + *p_left_separator_key = node_keys.front(); + } else { + // Move/rotate the largest key from left sibling to the current node 'through' the parent + BOOST_ASSERT( left_keys.back() < *p_left_separator_key ); + BOOST_ASSERT( *p_left_separator_key < node_keys.front() ); + node_keys.front() = std::move( *p_left_separator_key ); + *p_left_separator_key = std::move( left_keys.back() ); + + rshift_chldrn( node ); + insrt_child( node, 0, children( *p_left_sibling ).back(), node_slot ); + } + + p_left_sibling->num_vals--; + verify( *p_left_sibling ); + + final_node_original_keys_offset = 1; + + BOOST_ASSUME( node. num_vals == N::min_values ); + BOOST_ASSUME( p_left_sibling->num_vals >= N::min_values ); + } + // Borrow from right sibling if possible + else + if ( p_right_sibling && can_borrow( *p_right_sibling ) ) + { + verify( *p_right_sibling ); + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); + node.num_vals++; + auto const node_keys{ keys( node ) }; + if constexpr ( leaf_node_type ) { + // Move the smallest key from the right sibling to the current node + auto & leftmost_right_key{ keys( *p_right_sibling ).front() }; + BOOST_ASSUME( *p_right_separator_key == leftmost_right_key ); // yes we expect exact or bitwise equality for key-copies in inner nodes + node_keys.back() = std::move( leftmost_right_key ); + lshift_keys( *p_right_sibling ); + // adjust the separator key in the parent + *p_right_separator_key = leftmost_right_key; + } else { + // Move/rotate the smallest key from the right sibling to the current node 'through' the parent + + // no comparator in base classes :/ + //BOOST_ASSUME( le( parent.keys[ parent_child_idx ], p_right_sibling->keys[ 0 ] ) ); + //BOOST_ASSERT( le( *( node_keys.end() - 2 ), *p_right_separator_key ) ); + node_keys.back() = std::move( *p_right_separator_key ); + *p_right_separator_key = std::move( keys( *p_right_sibling ).front() ); + insrt_child( node, num_chldrn( node ) - 1, children( *p_right_sibling ).front(), node_slot ); + lshift_keys ( *p_right_sibling ); + lshift_chldrn( *p_right_sibling ); + } + + p_right_sibling->num_vals--; + verify( *p_right_sibling ); + + BOOST_ASSUME( node. num_vals == N::min_values ); + BOOST_ASSUME( p_right_sibling->num_vals >= N::min_values ); + } + // Merge with left or right sibling + else + { + if ( p_left_sibling ) { + verify( *p_left_sibling ); + BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); + // Merge node -> left sibling + final_node = node.left; + final_node_original_keys_offset = p_left_sibling->num_vals; + merge_right_into_left( *p_left_sibling, node, parent, left_separator_key_idx, parent_child_idx ); + } else { + verify( *p_right_sibling ); + BOOST_ASSUME( parent_key_idx == 0 ); + // no comparator in base classes :/ + //BOOST_ASSUME( leq( parent.keys[ parent_key_idx ], p_right_sibling->keys[ 0 ] ) ); + // Merge right sibling -> node + merge_right_into_left( node, *p_right_sibling, parent, right_separator_key_idx, static_cast( parent_child_idx + 1 ) ); + } + + // propagate underflow + auto & depth_{ this->hdr().depth_ }; + auto & root_ { this->hdr().root_ }; + if ( parent.is_root() ) [[ unlikely ]] + { + BOOST_ASSUME( root_ == slot_of( parent ) ); + BOOST_ASSUME( level == 1 ); + BOOST_ASSUME( depth_ > level ); // 'leaf root' should be handled directly by the special case in erase() + auto & root{ as( parent ) }; + BOOST_ASSUME( !!root.children[ 0 ] ); + if ( underflowed( root ) ) + { + root_ = root.children[ 0 ]; + bptree_base::node( root_ ).parent = {}; + --depth_; + free( root ); + } + } + else + if ( underflowed( parent ) ) + { + BOOST_ASSUME( level > 0 ); + BOOST_ASSUME( level < depth_ ); + handle_underflow( parent, level - 1 ); + } + } + + return { final_node, final_node_original_keys_offset }; + } // handle_underflow() + + root_node & root() noexcept { return as( bptree_base::root() ); } + root_node const & root() const noexcept { return const_cast( *this ).root(); } + + using bptree_base::free; + void free( leaf_node & leaf ) noexcept { bptree_base::free_leaf( leaf ); } + + using bptree_base::unlink_and_free_node; + void unlink_and_free_node( leaf_node & leaf, leaf_node & cached_left_sibling ) noexcept { bptree_base::unlink_and_free_leaf( leaf, cached_left_sibling ); } + + template + [[ gnu::sysv_abi ]] static + void move_keys + ( + N const & source, node_size_type src_begin, node_size_type src_end, + N & target, node_size_type tgt_begin + ) noexcept; + [[ gnu::sysv_abi ]] + void move_chldrn + ( + inner_node const & source, node_size_type src_begin, node_size_type src_end, + inner_node & target, node_size_type tgt_begin + ) noexcept; + + void insrt_child( inner_node & target, node_size_type const pos, node_slot const child_slot, node_slot const cached_target_slot ) noexcept + { + BOOST_ASSUME( cached_target_slot == slot_of( target ) ); + auto & child{ node( child_slot ) }; + children( target )[ pos ] = child_slot; + child.parent = cached_target_slot; + child.parent_child_idx = pos; + } + void insrt_child( inner_node & target, node_size_type const pos, node_slot const child_slot ) noexcept + { + insrt_child( target, pos, child_slot, slot_of( target ) ); + } + + [[ gnu::sysv_abi ]] + void merge_right_into_left + ( + leaf_node & __restrict left, leaf_node & __restrict right, + inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx + ) noexcept + { + auto constexpr min{ leaf_node::min_values }; + BOOST_ASSUME( right.num_vals >= min - 1 ); BOOST_ASSUME( right.num_vals <= min ); + BOOST_ASSUME( left .num_vals >= min - 1 ); BOOST_ASSUME( left .num_vals <= min ); + BOOST_ASSUME( left .right == slot_of( right ) ); + BOOST_ASSUME( right.left == slot_of( left ) ); + + std::ranges::move( keys( right ), &keys( left ).back() + 1 ); + left .num_vals += right.num_vals; + right.num_vals = 0; + BOOST_ASSUME( left.num_vals >= 2 * min - 2 ); + BOOST_ASSUME( left.num_vals <= 2 * min - 1 ); + lshift_keys ( parent, parent_key_idx ); + lshift_chldrn( parent, parent_child_idx ); + BOOST_ASSUME( parent.num_vals ); + parent.num_vals--; + + unlink_and_free_node( right, left ); + verify( left ); + verify( parent ); + } + [[ gnu::sysv_abi ]] + void merge_right_into_left + ( + inner_node & __restrict left, inner_node & __restrict right, + inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx + ) noexcept + { + auto constexpr min{ inner_node::min_values }; + BOOST_ASSUME( right.num_vals >= min - 1 ); BOOST_ASSUME( right.num_vals <= min ); + BOOST_ASSUME( left .num_vals >= min - 1 ); BOOST_ASSUME( left .num_vals <= min ); + + move_chldrn( right, 0, num_chldrn( right ), left, num_chldrn( left ) ); + auto & separator_key{ parent.keys[ parent_key_idx ] }; + left.num_vals += 1; + keys( left ).back() = std::move( separator_key ); + std::ranges::move( keys( right ), keys( left ).end() ); + + left.num_vals += right.num_vals; + BOOST_ASSUME( left.num_vals >= left.max_values - 1 ); BOOST_ASSUME( left.num_vals <= left.max_values ); + verify( left ); + unlink_and_free_node( right, left ); + + lshift_keys ( parent, parent_key_idx ); + lshift_chldrn( parent, parent_child_idx ); + BOOST_ASSUME( parent.num_vals ); + parent.num_vals--; + } + + static void verify( auto const & node ) noexcept + { + BOOST_ASSERT( std::ranges::adjacent_find( keys( node ) ) == keys( node ).end() ); + bptree_base::verify( node ); + } + +private: + [[ gnu::const, gnu::noinline ]] + static node_slot::value_type node_count_required_for_values( size_type const number_of_values ) noexcept + { + if ( number_of_values <= leaf_node::max_values ) + return ( number_of_values != 0 ); + auto const leaf_count{ static_cast( divide_up( number_of_values, /*assuming an 'optimistic' reserve, i.e. for bulk insert*/leaf_node::max_values ) ) }; + auto total_count{ leaf_count }; + auto current_level_count{ leaf_count }; + auto depth{ 1 }; + while ( current_level_count > 1 ) + { + current_level_count = divide_up( current_level_count, inner_node::min_children ); // pessimistic about inner node utilization + total_count += current_level_count; + ++depth; + } + // +1 since we use a 1-based depth index (instead of a 0-based where -1 + // is used to denote an empty tree) + auto const minimum_height{ static_cast( 1 + std::ceil( std::log( leaf_count ) / std::log( inner_node::max_children ) ) ) }; + auto const maximum_height{ static_cast( 1 + std::ceil( std::log( leaf_count ) / std::log( inner_node::min_children ) ) ) }; + BOOST_ASSUME( depth >= minimum_height ); + BOOST_ASSUME( depth <= maximum_height ); + [[ maybe_unused ]] + auto tree_structure_overhead{ total_count - leaf_count }; + return total_count; + } +}; // class bptree_base_wkey + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base_wkey::fwd_iterator +//////////////////////////////////////////////////////////////////////////////// + +template +class bptree_base_wkey::fwd_iterator + : + public base_iterator, + public iter_impl +{ +private: + using impl = iter_impl; + + using base_iterator::base_iterator; + +public: + constexpr fwd_iterator() noexcept = default; + + Key & operator*() const noexcept + { + auto & leaf{ static_cast( node() ) }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + return leaf.keys[ pos_.value_offset ]; + } + + std::span get_contiguous_span_and_move_to_next_node() noexcept + { + auto & leaf{ static_cast( node() ) }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + std::span const span{ &leaf.keys[ pos_.value_offset ], leaf.num_vals - pos_.value_offset }; + if ( leaf.right ) [[ likely ]] + { + pos_.node = leaf.right; + pos_.value_offset = 0; + } + return span; + } + + constexpr fwd_iterator & operator++() noexcept { return static_cast( base_iterator::operator++() ); } + constexpr fwd_iterator & operator--() noexcept { return static_cast( base_iterator::operator--() ); } + using impl::operator++; + using impl::operator--; +}; // class fwd_iterator + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base_wkey::ra_iterator +//////////////////////////////////////////////////////////////////////////////// + +template +class bptree_base_wkey::ra_iterator + : + public base_random_access_iterator, + public iter_impl +{ +private: friend class bptree_base_wkey; + using base_random_access_iterator::base_random_access_iterator; + + leaf_node & node() const noexcept { return static_cast( base_random_access_iterator::node() ); } + +public: + constexpr ra_iterator() noexcept = default; + + // TODO deduplicate this w/ fwd_iterator + Key & operator*() const noexcept + { + auto & leaf{ node() }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + return leaf.keys[ pos_.value_offset ]; + } + + std::span get_contiguous_span_and_move_to_next_node() noexcept + { + auto & leaf{ static_cast( node() ) }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + std::span const span{ &leaf.keys[ pos_.value_offset ], leaf.num_vals - pos_.value_offset }; + index_ += span.size(); + pos_.node = leaf.right; + pos_.value_offset = 0; + return span; + } + + ra_iterator & operator+=( difference_type const n ) noexcept { return static_cast( base_random_access_iterator::operator+=( n ) ); } + + ra_iterator & operator++( ) noexcept { return static_cast( base_random_access_iterator::operator++( ) ); } + ra_iterator operator++(int) noexcept { return static_cast( base_random_access_iterator::operator++(0) ); } + ra_iterator & operator--( ) noexcept { return static_cast( base_random_access_iterator::operator--( ) ); } + ra_iterator operator--(int) noexcept { return static_cast( base_random_access_iterator::operator--(0) ); } + + friend constexpr bool operator==( ra_iterator const & left, ra_iterator const & right ) noexcept { return left.index_ == right.index_; } + + operator fwd_iterator() const noexcept { return static_cast( static_cast( *this ) ); } +}; // class ra_iterator + +template +typename +bptree_base_wkey::iterator +bptree_base_wkey::erase( const_iterator const iter ) noexcept +{ + auto const [node, key_offset]{ iter.base().pos() }; + auto & lf{ leaf( node ) }; + if ( key_offset == 0 ) [[ unlikely ]] + update_separator( lf, lf.keys[ 1 ] ); + return make_iter( erase( lf, key_offset ) ); +} + +template +template [[ gnu::sysv_abi ]] +void bptree_base_wkey::move_keys +( + N const & source, node_size_type const src_begin, node_size_type const src_end, + N & target, node_size_type const tgt_begin +) noexcept +{ + BOOST_ASSUME( &source != &target ); // otherwise could require move_backwards or shift_* + BOOST_ASSUME( src_begin <= src_end ); + BOOST_ASSUME( ( src_end - src_begin ) <= N::max_values ); + BOOST_ASSUME( tgt_begin < N::max_values ); + std::uninitialized_move( &source.keys[ src_begin ], &source.keys[ src_end ], &target.keys[ tgt_begin ] ); +} +template [[ gnu::noinline, gnu::sysv_abi ]] +void bptree_base_wkey::move_chldrn +( + inner_node const & source, node_size_type const src_begin, node_size_type const src_end, + inner_node & target, node_size_type const tgt_begin +) noexcept +{ + BOOST_ASSUME( &source != &target ); // otherwise could require move_backwards or shift_* + BOOST_ASSUME( src_begin <= src_end ); + auto const count{ static_cast( src_end - src_begin ) }; + BOOST_ASSUME( count <= inner_node::min_children + 1 ); + BOOST_ASSUME( tgt_begin < inner_node::max_children ); + auto const src_chldrn{ &source.children[ src_begin ] }; + + auto const target_slot{ slot_of( target ) }; + for ( node_size_type ch_idx{ 0 }; ch_idx < count; ++ch_idx ) + { + auto & ch_slot{ src_chldrn[ ch_idx ] }; + auto & child { node( ch_slot ) }; + target.children[ tgt_begin + ch_idx ] = std::move( ch_slot ); + child.parent = target_slot; + child.parent_child_idx = tgt_begin + ch_idx; + } +} + + +//////////////////////////////////////////////////////////////////////////////// +// \class bp_tree +//////////////////////////////////////////////////////////////////////////////// + +template > +class bp_tree + : + public bptree_base_wkey, + private Comparator, + public boost::stl_interfaces::sequence_container_interface, boost::stl_interfaces::element_layout::discontiguous> +{ +private: + using base = bptree_base_wkey; + using stl_impl = boost::stl_interfaces::sequence_container_interface, boost::stl_interfaces::element_layout::discontiguous>; + + using depth_t = base::depth_t; + using node_size_type = base::node_size_type; + using key_const_arg = base::key_const_arg; + using node_slot = base::node_slot; + using node_header = base::node_header; + using leaf_node = base::leaf_node; + using root_node = base::root_node; + using inner_node = base::inner_node; + using parent_node = base::parent_node; + using fwd_iterator = base::fwd_iterator; + using ra_iterator = base:: ra_iterator; + using find_pos = base::find_pos; + using iter_pos = base::iter_pos; + + using bptree_base::as; + using bptree_base::can_borrow; + using bptree_base::children; + using bptree_base::keys; + using bptree_base::node; + using bptree_base::num_chldrn; + using bptree_base::lshift_keys; + using bptree_base::rshift_keys; + using bptree_base::lshift_chldrn; + using bptree_base::rshift_chldrn; + using bptree_base::slot_of; + using bptree_base::underflowed; + using bptree_base::verify; + + using base::free; + using base::insrt_child; + using base::leaf; + using base::leaf_level; + using base::parent; + using base::right; + using base::root; + +public: + static constexpr auto unique{ true }; // TODO non unique + static constexpr auto transparent_comparator{ requires{ typename Comparator::is_transparent; } }; + + using size_type = base::size_type; + using value_type = base::value_type; + using pointer = value_type *; + using const_pointer = value_type const *; + using reference = value_type &; + using const_reference = value_type const &; + using iterator = base:: iterator; + using const_iterator = base::const_iterator; + + using iter_pair = std::pair< iterator, iterator>; + using const_iter_pair = std::pair; + + using base::empty; + using base::size; + using base::clear; + + bp_tree() noexcept = default; + bp_tree( Comparator const & comp ) noexcept : Comparator{ comp } {} + + static constexpr base::size_type max_size() noexcept + { + auto const max_number_of_nodes { std::numeric_limits::max() }; + auto const max_number_of_leaf_nodes{ /*TODO*/ max_number_of_nodes }; + auto const values_per_node { leaf_node::max_values }; + auto const max_sz { max_number_of_leaf_nodes * values_per_node }; + static_assert( max_sz <= std::numeric_limits::max() ); + return max_sz; + } + + [[ gnu::pure ]] iterator begin() noexcept { return static_cast( base::begin() ); } using stl_impl::begin; + [[ gnu::pure ]] iterator end() noexcept { return static_cast( base:: end() ); } using stl_impl::end ; + + [[ gnu::pure ]] ra_iterator ra_begin() noexcept { return static_cast( base::ra_begin() ); } + [[ gnu::pure ]] ra_iterator ra_end () noexcept { return static_cast( base::ra_end () ); } + [[ gnu::pure ]] std::basic_const_iterator ra_begin() const noexcept { return const_cast( *this ).ra_begin(); } + [[ gnu::pure ]] std::basic_const_iterator ra_end () const noexcept { return const_cast( *this ).ra_end (); } + + auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } + auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } + + [[ nodiscard ]] bool contains ( LookupType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key } ); } + [[ nodiscard ]] iterator find ( LookupType auto const & key ) noexcept { return find_impl ( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iterator find ( LookupType auto const & key ) const noexcept { return const_cast( *this ).find( key ); } + [[ nodiscard ]] iterator lower_bound( LookupType auto const & key ) noexcept { return lower_bound_impl( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iterator lower_bound( LookupType auto const & key ) const noexcept { return const_cast( *this ).lower_bound( key ); } + [[ nodiscard ]] iter_pair equal_range( LookupType auto const & key ) noexcept { return equal_range_impl( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iter_pair equal_range( LookupType auto const & key ) const noexcept { return const_cast( *this ).equal_range( key ); } + + std::pair insert( InsertableType auto const & key ) { return insert_impl( pass_in_reg{ key } ); } + iterator insert( const_iterator const pos_hint, InsertableType auto const & key ) { return insert_impl( pos_hint, pass_in_reg{ key } ); } + + // bulk insert + // performance note: insertion of existing values into a unique bp_tree is + // supported and accounted for (the input values are skipped) but it is + // considered an 'unlikely' event and as such it is handled by sad/cold paths + // TODO complete std insert interface (w/ ranges, iterators, hints...) + template + size_type insert( InIter const begin, InIter const end ) { return insert( base::bulk_insert_prepare( std::ranges::subrange( begin, end ) ) ); } + size_type insert( std::ranges::range auto const & keys ) { return insert( base::bulk_insert_prepare( std::ranges::subrange( keys ) ) ); } + + size_type merge( bp_tree && other ); + + using base::erase; + [[ nodiscard ]] BOOST_NOINLINE + bool erase( key_const_arg key ) noexcept + { + auto const location{ find_nodes_for( key ) }; + + if ( !location.leaf_offset.exact_find ) [[ unlikely ]] + return false; + + leaf_node & leaf{ location.leaf }; + if ( this->hdr().depth_ != 1 ) + { + verify( leaf ); + BOOST_ASSUME( leaf.num_vals >= leaf.min_values ); + } + auto const leaf_key_offset{ location.leaf_offset.pos }; + if ( location.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" + { + BOOST_ASSUME( leaf_key_offset == 0 ); + BOOST_ASSUME( eq( leaf.keys[ leaf_key_offset ], key ) ); + + auto & inner { node( location.inner ) }; + auto & separator_key{ inner.keys[ location.inner_offset ] }; + BOOST_ASSUME( eq( separator_key, key ) ); + BOOST_ASSUME( leaf_key_offset + 1 < leaf.num_vals ); + separator_key = leaf.keys[ leaf_key_offset + 1 ]; + } + + base::erase( leaf, leaf_key_offset ); + return true; + } + + void swap( bp_tree & other ) noexcept { base::swap( other ); } + + [[ nodiscard ]] Comparator const & comp() const noexcept { return *this; } + + // UB if the comparator is changed in such a way as to invalidate to order of elements already in the container + [[ nodiscard ]] Comparator & mutable_comp() noexcept { return *this; } + +private: // pass-in-reg public function overloads/impls + bool contains_impl( Reg auto const key ) const noexcept { return !empty() && const_cast( *this ).find_nodes_for( key ).leaf_offset.exact_find; } + + [[ using gnu: pure, sysv_abi ]] + iterator find_impl( Reg auto const key ) noexcept + { + if ( !empty() ) [[ likely ]] + { + auto const location{ find_nodes_for( key ) }; + if ( location.leaf_offset.exact_find ) [[ likely ]] { + return base::make_iter( location ); + } + } + + return this->end(); + } + [[ using gnu: pure, sysv_abi ]] + iterator lower_bound_impl( Reg auto const key ) noexcept + { + if ( !empty() ) [[ likely ]] + { + auto const location{ find_nodes_for( key ) }; + // find_nodes_for() returns an insertion point, which can be one + // pass the end of a node (like an end iterator, indicating an + // append/push_back position) but iterators do not support this + // (have to be initialized with a valid position for the increment + // operator to work). + if ( location.leaf_offset.pos == location.leaf.num_vals ) [[ unlikely ]] { + BOOST_ASSUME( !location.leaf_offset.exact_find ); + return base::make_iter( location.leaf.right, node_size_type{ 0 } ); + } + return base::make_iter( location ); + } + + return end(); + } + + [[ using gnu: pure, sysv_abi ]] + iter_pair equal_range_impl( Reg auto const key ) noexcept + { + if ( !empty() ) [[ likely ]] + { + auto const location{ find_nodes_for( key ) }; + if ( location.leaf_offset.exact_find ) [[ likely ]] { + auto const begin{ base::make_iter( location ) }; + return { begin, std::next( begin ) }; + } + } + + auto const end_iter{ end() }; + return { end_iter, end_iter }; + } + + std::pair insert_impl( Reg auto const v ) + { + if ( empty() ) + { + auto & root{ static_cast( base::create_root() ) }; + BOOST_ASSUME( root.num_vals == 1 ); + root.keys[ 0 ] = v; + return { begin(), true }; + } + + auto const locations{ find_nodes_for( v ) }; + BOOST_ASSUME( !locations.inner ); + BOOST_ASSUME( !locations.inner_offset ); + if ( locations.leaf_offset.exact_find ) [[ unlikely ]] + return { base::make_iter( locations ), false }; + + auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; + ++this->hdr().size_; + return { base::make_iter( insert_pos_next ), true }; + } + + iterator insert_impl( const_iterator const pos_hint, Reg auto const v ) + { + // yes, for starters, generic 'hint as just a hint' is not supported + BOOST_ASSUME( !empty() ); + BOOST_ASSERT_MSG( le( v, *pos_hint ), "Invalid insertion hint" ); + BOOST_ASSERT_MSG( !unique || ge( v, *std::prev( pos_hint ) ), "Invalid insertion hint" ); + + auto const [hint_slot, hint_slot_offset]{ pos_hint.base().pos() }; + auto & hint_leaf{ leaf( hint_slot ) }; + if ( hint_slot_offset == 0 ) [[ unlikely ]] { + base::update_separator( hint_leaf, v ); + } + [[ maybe_unused ]] + auto const insert_pos_next{ base::insert( hint_leaf, hint_slot_offset, Key{ v }, {} ) }; + BOOST_ASSERT( pos_hint == base::make_iter( insert_pos_next ) ); + ++this->hdr().size_; + return pos_hint.base(); + } + +private: + // lower_bound find + [[ using gnu: pure, hot, noinline, sysv_abi ]] + static find_pos find( Key const keys[], node_size_type const num_vals, Reg auto const value, pass_in_reg const comparator ) noexcept + { + // TODO branchless binary search, Alexandrescu's TLC, + // https://orlp.net/blog/bitwise-binary-search + // https://algorithmica.org/en/eytzinger + // FAST: Fast Architecture Sensitive Tree Search on Modern CPUs and GPUs http://kaldewey.com/pubs/FAST__SIGMOD10.pdf + // ... + BOOST_ASSUME( num_vals > 0 ); + BOOST_ASSUME( num_vals <= leaf_node::max_values ); + Comparator const & __restrict comp( comparator ); + if constexpr ( use_linear_search_for_sorted_array ) + { + for ( node_size_type k{ 0 }; ; ) + { + if ( !comp( keys[ k ], value ) ) + { + auto const exact_find{ !comp( value, keys[ k ] ) }; + return { k, exact_find }; + } + if ( ++k == num_vals ) + { + return { k, false }; + } + } + } + else + { + auto const pos_iter { std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp ) }; + auto const pos_idx { static_cast( std::distance( &keys[ 0 ], pos_iter ) ) }; + auto const exact_find{ ( pos_idx != num_vals ) && !comp( value, keys[ pos_idx ] ) }; + return { pos_idx, exact_find }; + } + std::unreachable(); + } + find_pos find( Key const keys[], node_size_type const num_vals, Reg auto const value ) const noexcept { return find( keys, num_vals, value, pass_in_reg{ comp() } ); } + find_pos find( auto const & node, auto const & value ) const noexcept { return find( node.keys, node.num_vals, pass_in_reg{ value } ); } + [[ using gnu: pure, hot, sysv_abi ]] + find_pos find_with_offset( auto const & node, node_size_type const offset, Reg auto const value ) const noexcept + { + BOOST_ASSUME( offset < node.num_vals ); + auto result{ find( &node.keys[ offset ], node.num_vals - offset, value ) }; + result.pos += offset; + return result; + } + + [[ using gnu: pure, hot, sysv_abi, noinline ]] + base::key_locations find_nodes_for( Reg auto const key ) noexcept + { + node_slot separator_key_node; + node_size_type separator_key_offset{}; + // a leaf (lone) root is implicitly handled by the loop condition: + // depth_ == 1 so the loop is skipped entirely and the lone root is + // never examined through the incorrectly typed reference + auto p_node{ &as( root() ) }; + auto const depth { this->hdr().depth_ }; + BOOST_ASSUME( depth >= 1 ); + for ( auto level{ 0 }; level < depth - 1; ++level ) + { + auto [pos, exact_find]{ find( *p_node, key ) }; + if ( exact_find ) + { + // separator key - it also means we have to traverse to the right + BOOST_ASSUME( !separator_key_node ); // exact_find may happen at most once + separator_key_node = slot_of( *p_node ); + separator_key_offset = pos; + ++pos; // traverse to the right child + } + p_node = &node( p_node->children[ pos ] ); + } + auto & leaf{ base::template as( *p_node ) }; + return + { + leaf, + separator_key_node ? find_pos{ 0, true } : find( leaf, key ), + separator_key_offset, + separator_key_node + }; + } + auto find_nodes_for( Key const & key ) noexcept { return find_nodes_for( key ); } + + auto find_next( leaf_node const & starting_leaf, node_size_type const starting_leaf_offset, Reg auto const key ) const noexcept + { + if ( leq( key, keys( starting_leaf ).back() ) ) + { + auto const pos{ find_with_offset( starting_leaf, starting_leaf_offset, key ) }; + BOOST_ASSUME( pos.pos != starting_leaf.num_vals ); + BOOST_ASSUME( pos.pos >= starting_leaf_offset ); + return std::make_pair( const_cast( &starting_leaf ), pos ); + } + + if ( !starting_leaf.right ) [[ unlikely ]] // we are at the end of the tree/leaf level: key not present at all + return std::make_pair( const_cast( &starting_leaf ), find_pos{ starting_leaf.num_vals, false } ); + + // key in tree but not in starting leaf: go up the tree + auto const * prnt{ &parent( starting_leaf ) }; + auto parent_offset{ starting_leaf.parent_child_idx }; + BOOST_ASSUME( ( parent_offset == prnt->num_vals ) || ge( prnt->keys[ parent_offset ], starting_leaf.keys[ 0 ] ) ); + auto const depth{ this->hdr().depth_ }; BOOST_ASSUME( depth >= 1 ); + auto level{ depth - 1 }; + while ( le( keys( *prnt ).back(), key ) ) + { + if ( level == 1 ) [[ unlikely ]] + { + // reached the root + BOOST_ASSUME( !prnt->parent ); + // the case where the key does not exist at all is handled at + // the beginning so the only case where parent_offset could + // point to the end is on intermediate inner/parent nodes (when + // depth is more then 2 levels) + // (and this has to be handled because otherwise the + // find_with_offset call below would get fed empty input which + // it does not support) + BOOST_ASSUME( depth > 2 || ( parent_offset < prnt->num_vals ) ); + parent_offset = std::min( parent_offset, node_size_type( prnt->num_vals - 1 ) ); + break; + } + parent_offset = prnt->parent_child_idx; + prnt = &parent( *prnt ); + --level; + } + BOOST_ASSUME( parent_offset < prnt->num_vals ); + // descend to the leaf containing the key + for ( ; level < depth; ++level ) + { + auto [pos, exact_find]{ find_with_offset( *prnt, parent_offset, key ) }; + BOOST_ASSERT( !exact_find ); + pos += exact_find; // traverse to the right child for separator keys + prnt = &node( children( *prnt )[ pos ] ); + parent_offset = 0; + } + BOOST_ASSUME( parent_offset == 0 ); + auto const & containing_leaf{ as( *prnt ) }; + auto const pos{ find( containing_leaf, key ) }; + BOOST_ASSUME + ( + ( &starting_leaf != &containing_leaf ) || + // the worst case: when the value falls between existing nodes we + // will land on the starting node again - TODO insert a new node + ( pos.pos == containing_leaf.num_vals ) + ); + return std::make_pair( const_cast( &containing_leaf ), pos ); + } + + template + void insert( N & target_node, key_const_arg v, node_slot const right_child ) + { + auto const pos{ find( target_node, v ).pos }; + base::insert( target_node, pos, v, right_child ); + } + + size_type insert( typename base::bulk_copied_input ); + + // bulk insert helper: merge a new, presorted leaf into an existing leaf + auto merge + ( + leaf_node const & source, node_size_type source_offset, + leaf_node & target, node_size_type target_offset + ) noexcept; + + node_size_type merge_interleaved_values + ( + Key const source0[], node_size_type const source0_size, + Key const source1[], node_size_type const source1_size, + Key target [] + ) const noexcept + { + node_size_type const input_size( source0_size + source1_size ); + if constexpr ( unique ) + { + auto const out_pos{ std::set_union( source0, &source0[ source0_size ], source1, &source1[ source1_size ], target, comp() ) }; + auto const merged_size{ static_cast( out_pos - target ) }; + BOOST_ASSUME( merged_size <= input_size ); + return merged_size; + } + else + { + auto const out_pos{ std::merge( source0, &source0[ source0_size ], source1, &source1[ source1_size ], target, comp() ) }; + BOOST_ASSUME( out_pos = target + input_size ); + return input_size; + } + } + +#if !( defined( _MSC_VER ) && !defined( __clang__ ) ) + // ambiguous call w/ VS 17.11.5 + void verify( auto const & node ) const noexcept + { + BOOST_ASSERT( std::ranges::is_sorted( keys( node ), comp() ) ); + base::verify( node ); + } +#endif + + [[ gnu::pure ]] bool le( key_const_arg left, key_const_arg right ) const noexcept { return comp()( left, right ); } + [[ gnu::pure ]] bool ge( key_const_arg left, key_const_arg right ) const noexcept { return comp()( right, left ); } + [[ gnu::pure ]] bool eq( key_const_arg left, key_const_arg right ) const noexcept + { + if constexpr ( requires{ comp().eq( left, right ); } ) + return comp().eq( left, right ); + if constexpr ( is_simple_comparator && requires{ left == right; } ) + return left == right; + return !comp()( left, right ) && !comp()( right, left ); + } + [[ gnu::pure ]] bool leq( key_const_arg left, key_const_arg right ) const noexcept + { + if constexpr ( requires{ comp().leq( left, right ); } ) + return comp().leq( left, right ); + return !comp()( right, left ); + } + [[ gnu::pure ]] bool geq( key_const_arg left, key_const_arg right ) const noexcept + { + if constexpr ( requires{ comp().geq( left, right ); } ) + return comp().geq( left, right ); + return !comp()( left, right ); + } +}; // class bp_tree + +PSI_WARNING_DISABLE_POP() + + +template +// bulk insert helper: merge a new, presorted leaf into an existing leaf +auto bp_tree::merge +( + leaf_node const & source, node_size_type const source_offset, + leaf_node & target, node_size_type const target_offset +) /*?*/noexcept +{ + verify( source ); + verify( target ); + BOOST_ASSUME( source_offset < source.num_vals ); + node_size_type input_length ( source.num_vals - source_offset ); + node_size_type const available_space( target.max_values - target.num_vals ); + auto const src_keys{ &source.keys[ source_offset ] }; + auto & tgt_keys{ target.keys }; + if ( target_offset == 0 ) [[ unlikely ]] + { + auto const & new_separator{ src_keys[ 0 ] }; + BOOST_ASSUME( new_separator < tgt_keys[ 0 ] ); + base::update_separator( target, new_separator ); + // TODO rather simply insert the source leaf into the parent (if all of + // its keys come before the first key in target) + } + if ( !available_space ) [[ unlikely ]] + { + // support merging nodes from another tree instance + auto const source_slot{ base::is_my_node( source ) ? slot_of( source ) : node_slot{} }; + BOOST_ASSERT( find( target, src_keys[ 0 ] ).pos == target_offset ); + if ( eq( target.keys[ target_offset ], src_keys[ 0 ] ) ) [[ unlikely ]] + { + return std::make_tuple( 0, 1, &target, target_offset ); + } + auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, pass_rv_in_reg{ /*mrmlj*/Key{ src_keys[ 0 ] } }, {} ) }; + auto const & src{ source_slot ? leaf( source_slot ) : source }; + auto & tgt{ leaf( target_slot ) }; + BOOST_ASSUME( next_tgt_offset <= tgt.num_vals ); + auto const next_src_offset{ static_cast( source_offset + 1 ) }; + // next_tgt_offset returned by split_to_insert points to the + // position in the target node that immediately follows the + // position for the inserted src_keys[ 0 ] - IOW it need not be + // the position for src.keys[ next_src_offset ] + if + ( + ( next_tgt_offset != tgt.num_vals ) && // necessary check because find assumes non-empty input + ( next_src_offset != src.num_vals ) && // possible edge case where there was actually only one key left in the source + false // not really worth it: the caller still has to call find on/for returns from all the other branches + ) + { + next_tgt_offset = find_with_offset( tgt, next_tgt_offset, key_const_arg{ src.keys[ next_src_offset ] } ).pos; + } + return std::make_tuple( 1, 1, &tgt, next_tgt_offset ); + } + + auto copy_size{ std::min( input_length, available_space ) }; + // If there is an existing right sibling we must first check if the + // source contains values beyond its separator key (and adjust the copy + // size accordingly to maintain the sorted property). + if ( target.right ) + { + auto const & right_delimiter { right( target ).keys[ 0 ] }; + auto const less_than_right_pos{ find( src_keys, copy_size, key_const_arg{ right_delimiter } ) }; + BOOST_ASSUME( !less_than_right_pos.exact_find ); + if ( less_than_right_pos.pos != copy_size ) + { + BOOST_ASSUME( less_than_right_pos.pos < copy_size ); + node_size_type const input_end_for_target( less_than_right_pos.pos + source_offset ); + BOOST_ASSUME( input_end_for_target > source_offset ); + BOOST_ASSUME( input_end_for_target <= source.num_vals ); + copy_size = static_cast( input_end_for_target - source_offset ); + } + } + + auto & tgt_size{ target.num_vals }; + node_size_type inserted_size; + node_size_type next_tgt_offset; + if ( target_offset == tgt_size ) // a simple append + { + std::copy_n( src_keys, copy_size, &tgt_keys[ target_offset ] ); + tgt_size += copy_size; + inserted_size = copy_size; + next_tgt_offset = tgt_size; + } + else + { + BOOST_ASSUME( copy_size + tgt_size <= leaf_node::max_values ); + // make room for merge: move existing values (beyond the insertion/merge + // point) to the end of the buffer + std::move_backward( &tgt_keys[ target_offset ], &tgt_keys[ tgt_size ], &tgt_keys[ tgt_size + copy_size ] ); + auto const new_tgt_size{ target_offset + merge_interleaved_values + ( + &src_keys[ 0 ], copy_size, + &tgt_keys[ target_offset + copy_size ], tgt_size - target_offset, + &tgt_keys[ target_offset ] + ) }; + inserted_size = static_cast( new_tgt_size - tgt_size ); + tgt_size = static_cast( new_tgt_size ); + next_tgt_offset = target_offset + 1; + } + verify( target ); + BOOST_ASSUME( inserted_size <= copy_size ); + return std::make_tuple( inserted_size, copy_size, &target, next_tgt_offset ); +} + + +template +bp_tree::size_type +bp_tree::insert( typename base::bulk_copied_input const input ) +{ + // https://www.sciencedirect.com/science/article/abs/pii/S0020025502002025 On batch-constructing B+-trees: algorithm and its performance + // https://www.vldb.org/conf/2001/P461.pdf An Evaluation of Generic Bulk Loading Techniques + // https://stackoverflow.com/questions/15996319/is-there-any-algorithm-for-bulk-loading-in-b-tree + + auto const [begin_leaf, end_pos, total_size]{ input }; + ra_iterator const p_new_nodes_begin{ *this, { begin_leaf, 0 }, 0 }; + ra_iterator const p_new_nodes_end { *this, end_pos , total_size }; +#if 0 // slower + std::sort( p_new_nodes_begin, p_new_nodes_end, comp() ); +#else + boost::movelib::pdqsort( p_new_nodes_begin, p_new_nodes_end, comp() ); +#endif + + if ( empty() ) + { + base::bulk_insert_into_empty( begin_leaf, end_pos, total_size ); + BOOST_ASSUME( this->hdr().size_ == total_size ); + return total_size; + } + + auto p_new_keys{ p_new_nodes_begin }; + auto [source_slot, source_slot_offset]{ p_new_keys.pos() }; + auto src_leaf{ &leaf( source_slot ) }; + + auto const start_pos{ find_nodes_for( *p_new_keys ) }; + auto tgt_leaf { &start_pos.leaf }; + auto tgt_leaf_next_pos{ start_pos.leaf_offset }; + + size_type inserted{ 0 }; + do + { + if ( unique && tgt_leaf_next_pos.exact_find ) [[ unlikely ]] + { + ++p_new_keys; + continue; + } + + BOOST_ASSUME( source_slot_offset < src_leaf->num_vals ); + // if we have reached the end of the rightmost leaf simply perform + // a bulk_append + if ( ( tgt_leaf_next_pos.pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) + { + auto const so_far_consumed{ static_cast( p_new_keys - p_new_nodes_begin ) }; + BOOST_ASSUME( so_far_consumed < total_size ); + std::shift_left( &src_leaf->keys[ 0 ], &src_leaf->keys[ src_leaf->num_vals ], source_slot_offset ); + src_leaf->num_vals -= source_slot_offset; + base::link( *tgt_leaf, *src_leaf ); + base::bulk_append_fill_incomplete_leaf( *src_leaf ); // yes, in case src_leaf is really incomplete, the shift_left above is redundant + auto const rightmost_parent_slot{ tgt_leaf->parent }; + auto const parent_pos { tgt_leaf->parent_child_idx }; // key idx = child idx - 1 & this is the 'next' key + base::bulk_append( src_leaf, { rightmost_parent_slot, parent_pos } ); + inserted += static_cast( total_size - so_far_consumed ); + break; + } + // TODO in-the-middle partial bulk-inserts + + auto const [inserted_count, consumed_source, tgt_next_leaf, tgt_next_offset] + { + merge + ( + *src_leaf, source_slot_offset, + *tgt_leaf, tgt_leaf_next_pos.pos + ) + }; + tgt_leaf = tgt_next_leaf; + + // merge might have caused a relocation (by calling split_to_insert) + // TODO use iter_pos directly + p_new_keys .update_pool_ptr( this->nodes_ ); + p_new_nodes_end.update_pool_ptr( this->nodes_ ); + src_leaf = &leaf( source_slot ); + + p_new_keys += consumed_source; + inserted += inserted_count; + + if ( source_slot != p_new_keys.pos().node ) // have we moved to the next node? + { + // merged leaves (their contents) were effectively copied into + // existing leaves (instead of simply linked into the tree + // structure) and now have to be returned to the free pool + // TODO: add leak detection to/for the entire bp_tree class + base::unlink_right( *src_leaf ); + free( *src_leaf ); + + source_slot = p_new_keys.pos().node; + src_leaf = &leaf( source_slot ); + } + source_slot_offset = p_new_keys.pos().value_offset; + + // seek the next position starting from the current one (relying on the + // that we are using presorted data) rather than starting everytime from + // scratch (using find_nodes_for) + std::tie( tgt_leaf, tgt_leaf_next_pos ) = + find_next( *tgt_leaf, tgt_next_offset, key_const_arg{ src_leaf->keys[ source_slot_offset ] } ); + } while ( p_new_keys != p_new_nodes_end ); + + BOOST_ASSUME( inserted <= total_size ); + this->hdr().size_ += inserted; + return inserted; +} // bp_tree::insert() + +template +bp_tree::size_type +bp_tree::merge( bp_tree && other ) +{ + // This function follows nearly the same logic as bulk insert (consult it + // for more comments), the main differences being: + // - no need to copy and sort the input + // - the bulk_append phase has to first copy the remainder of the source + // nodes (they are not somehow 'extractable' from the source tree) + // - care has to be taken around the fact that source leaves are coming + // from a different tree instance (i.e. from a different container) e.g. + // when resolving slots to node references. + // TODO further deduplicate with insert + if ( empty() ) { + swap( other ); + return size(); + } + + auto const total_size{ other.size() }; + this->reserve_additional( total_size ); + + auto const p_new_nodes_begin{ other.ra_begin() }; + auto const p_new_nodes_end { other.ra_end () }; + + auto p_new_keys{ p_new_nodes_begin }; + auto src_leaf { &other.leaf( p_new_keys.pos().node ) }; + auto source_slot_offset{ p_new_keys.pos().value_offset }; + + auto const start_pos{ find_nodes_for( *p_new_keys ) }; + auto tgt_leaf { &start_pos.leaf }; + auto tgt_leaf_next_pos{ start_pos.leaf_offset }; + + size_type inserted{ 0 }; + do + { + if ( unique && tgt_leaf_next_pos.exact_find ) [[ unlikely ]] + { + ++p_new_keys; + continue; + } + + BOOST_ASSUME( source_slot_offset < src_leaf->num_vals ); + // simple bulk_append at the end of the rightmost leaf + if ( ( tgt_leaf_next_pos.pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) + { + // pre-copy the (remainder of the) source into fresh nodes in + // order to simply call bulk_append + node_slot src_copy_begin; + node_slot prev_src_copy_node; + for ( ;; ) + { + auto & src_leaf_copy{ this->template new_node() }; + if ( !src_copy_begin ) + { + src_copy_begin = slot_of( src_leaf_copy ); + this->move_keys( *src_leaf, source_slot_offset, src_leaf->num_vals, src_leaf_copy, 0 ); + src_leaf_copy.num_vals = src_leaf->num_vals - source_slot_offset; + src_leaf->num_vals = source_slot_offset; + this->link( *tgt_leaf, src_leaf_copy ); + this->bulk_append_fill_incomplete_leaf( src_leaf_copy ); + } + else + { + this->move_keys( *src_leaf, 0, src_leaf->num_vals, src_leaf_copy, 0 ); + src_leaf_copy.num_vals = src_leaf->num_vals; + src_leaf->num_vals = 0; + this->link( this->leaf( prev_src_copy_node ), src_leaf_copy ); + } + BOOST_ASSUME( !src_leaf_copy.parent ); + BOOST_ASSUME( !src_leaf_copy.parent_child_idx ); + + if ( !src_leaf->right ) + break; + src_leaf = &other.right( *src_leaf ); + prev_src_copy_node = slot_of( src_leaf_copy ); + } + + auto const so_far_consumed{ static_cast( p_new_keys - p_new_nodes_begin ) }; + BOOST_ASSUME( so_far_consumed < total_size ); + this->bulk_append( &this->leaf( src_copy_begin ), { tgt_leaf->parent, tgt_leaf->parent_child_idx } ); + inserted += static_cast( total_size - so_far_consumed ); + break; + } + // TODO in-the-middle partial bulk-inserts + + auto const [inserted_count, consumed_source, tgt_next_leaf, tgt_next_offset] + { + merge + ( + *src_leaf, source_slot_offset, + *tgt_leaf, tgt_leaf_next_pos.pos + ) + }; + tgt_leaf = tgt_next_leaf; + + p_new_keys += consumed_source; + inserted += inserted_count; + + src_leaf = &other.leaf( p_new_keys.pos().node ); + source_slot_offset = p_new_keys.pos().value_offset; + + std::tie( tgt_leaf, tgt_leaf_next_pos ) = + find_next( *tgt_leaf, tgt_next_offset, pass_in_reg{ src_leaf->keys[ source_slot_offset ] } ); + } while ( p_new_keys != p_new_nodes_end ); + + BOOST_ASSUME( inserted <= total_size ); + this->hdr().size_ += inserted; + return inserted; +} // bp_tree::merge() + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/b+tree_print.hpp b/include/psi/vm/containers/b+tree_print.hpp new file mode 100644 index 0000000..1042b21 --- /dev/null +++ b/include/psi/vm/containers/b+tree_print.hpp @@ -0,0 +1,83 @@ +#pragma once + +#include "b+tree.hpp" + +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +template +void bptree_base_wkey::print() const +{ + if ( empty() ) + { + std::puts( "The tree is empty." ); + return; + } + + // BFS, one level of the tree at a time. + auto p_node{ &as( root() ) }; + for ( depth_t level{ 0 }; !is_leaf_level( level ); ++level ) + { + std::print( "Level {}:\t", std::uint16_t( level ) ); + + auto p_next_level{ &node( children( *p_node ).front() ) }; + + std::uint32_t level_node_count{ 0 }; + size_type level_key_count { 0 }; + // Process all nodes at the current level + for ( ; ; ) + { + // Internal node, print keys and add children to the queue + level_key_count += num_vals( *p_node ); + std::putchar( '<' ); + for ( auto i{ 0U }; i < num_vals( *p_node ); ++i ) + { + std::print( "{}", keys( *p_node )[ i ] ); + if ( i < num_vals( *p_node ) - 1U ) + std::print( ", " ); + } + std::print( "> " ); + + ++level_node_count; + if ( !p_node->right ) + break; + p_node = &node( p_node->right ); + } + std::println( " [{} nodes w/ {} values]", level_node_count, level_key_count ); + + p_node = p_next_level; + } + + { + std::print( "Leaf level ({}):\t", leaf_level() ); + std::uint32_t level_node_count{ 0 }; + size_type level_key_count { 0 }; + auto p_leaf{ &as( *p_node ) }; + for ( ; ; ) + { + level_key_count += num_vals( *p_leaf ); + std::putchar( '[' ); + for ( auto i{ 0U }; i < num_vals( *p_leaf ); ++i ) { + std::print( "{}", keys( *p_leaf )[ i ] ); + if ( i < num_vals( *p_leaf ) - 1U ) { + std::print( ", " ); + } + } + std::print( "] " ); + + ++level_node_count; + if ( !p_leaf->right ) + break; + p_leaf = &node( p_leaf->right ); + } + std::println( " [{} nodes w/ {} values]", level_node_count, level_key_count ); + } +} + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/detail/nt.hpp b/include/psi/vm/detail/nt.hpp index b4f57df..fc2e465 100644 --- a/include/psi/vm/detail/nt.hpp +++ b/include/psi/vm/detail/nt.hpp @@ -30,13 +30,18 @@ namespace psi::vm::nt { //------------------------------------------------------------------------------ +// TODO move to https://github.com/winsiderss/phnt + #ifndef STATUS_SUCCESS -using NTSTATUS = DWORD; +using NTSTATUS = LONG; NTSTATUS constexpr STATUS_SUCCESS{ 0 }; #endif // STATUS_SUCCESS #ifndef STATUS_CONFLICTING_ADDRESSES auto constexpr STATUS_CONFLICTING_ADDRESSES{ NTSTATUS( 0xC0000018 ) }; #endif +#ifndef STATUS_SECTION_NOT_EXTENDED +auto constexpr STATUS_SECTION_NOT_EXTENDED{ NTSTATUS( 0xC0000087 ) }; +#endif inline auto const current_process{ reinterpret_cast( std::intptr_t{ -1 } ) }; @@ -63,7 +68,7 @@ using NtCreateSection_t = NTSTATUS (NTAPI*) IN ULONG PageAttributess, IN ULONG SectionAttributes, IN HANDLE FileHandle OPTIONAL -); +) noexcept; inline auto const NtCreateSection{ detail::get_nt_proc( "NtCreateSection" ) }; enum SECTION_INFORMATION_CLASS { SectionBasicInformation, SectionImageInformation }; @@ -82,14 +87,14 @@ using NtQuerySection_t = NTSTATUS (NTAPI*) OUT PVOID InformationBuffer, IN ULONG InformationBufferSize, OUT PULONG ResultLength OPTIONAL -); +) noexcept; inline auto const NtQuerySection{ detail::get_nt_proc( "NtQuerySection" ) }; using NtExtendSection_t = NTSTATUS (NTAPI*) ( IN HANDLE SectionHandle, IN PLARGE_INTEGER NewSectionSize -); +) noexcept; inline auto const NtExtendSection{ detail::get_nt_proc( "NtExtendSection" ) }; enum SECTION_INHERIT @@ -109,12 +114,44 @@ using NtMapViewOfSection_t = NTSTATUS (NTAPI*) IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType OPTIONAL, IN ULONG Protect -); +) noexcept; inline auto const NtMapViewOfSection{ detail::get_nt_proc( "NtMapViewOfSection" ) }; -using NtAllocateVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, ULONG_PTR ZeroBits, PSIZE_T RegionSize, ULONG allocation_type, ULONG Protect ); -using NtFreeVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN PVOID * BaseAddress, PSIZE_T RegionSize, ULONG FreeType ); -using NtProtectVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, IN OUT PULONG NumberOfBytesToProtect, IN ULONG NewAccessProtection, OUT PULONG OldAccessProtection ); +enum MEMORY_INFORMATION_CLASS +{ + MemoryBasicInformation, // q: MEMORY_BASIC_INFORMATION + MemoryWorkingSetInformation, // q: MEMORY_WORKING_SET_INFORMATION + MemoryMappedFilenameInformation, // q: UNICODE_STRING + MemoryRegionInformation, // q: MEMORY_REGION_INFORMATION + MemoryWorkingSetExInformation, // q: MEMORY_WORKING_SET_EX_INFORMATION // since VISTA + MemorySharedCommitInformation, // q: MEMORY_SHARED_COMMIT_INFORMATION // since WIN8 + MemoryImageInformation, // q: MEMORY_IMAGE_INFORMATION + MemoryRegionInformationEx, // MEMORY_REGION_INFORMATION + MemoryPrivilegedBasicInformation, // MEMORY_BASIC_INFORMATION + MemoryEnclaveImageInformation, // MEMORY_ENCLAVE_IMAGE_INFORMATION // since REDSTONE3 + MemoryBasicInformationCapped, // 10 + MemoryPhysicalContiguityInformation, // MEMORY_PHYSICAL_CONTIGUITY_INFORMATION // since 20H1 + MemoryBadInformation, // since WIN11 + MemoryBadInformationAllProcesses, // since 22H1 + MemoryImageExtensionInformation, // MEMORY_IMAGE_EXTENSION_INFORMATION // since 24H2 + MaxMemoryInfoClass +}; + +using NtQueryVirtualMemory_t = NTSTATUS (NTAPI *) +( + HANDLE ProcessHandle, + PVOID BaseAddress, + MEMORY_INFORMATION_CLASS MemoryInformationClass, + PVOID MemoryInformation, + SIZE_T MemoryInformationLength, + PSIZE_T ReturnLength +) noexcept; +inline auto const NtQueryVirtualMemory{ detail::get_nt_proc( "NtQueryVirtualMemory" ) }; + + +using NtAllocateVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, ULONG_PTR ZeroBits, PSIZE_T RegionSize, ULONG allocation_type, ULONG Protect ) noexcept; +using NtFreeVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN PVOID * BaseAddress, PSIZE_T RegionSize, ULONG FreeType ) noexcept; +using NtProtectVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, IN OUT PULONG NumberOfBytesToProtect, IN ULONG NewAccessProtection, OUT PULONG OldAccessProtection ) noexcept; inline auto const NtAllocateVirtualMemory{ detail::get_nt_proc( "NtAllocateVirtualMemory" ) }; inline auto const NtFreeVirtualMemory { detail::get_nt_proc( "NtFreeVirtualMemory" ) }; diff --git a/include/psi/vm/flags/mapping.hpp b/include/psi/vm/flags/mapping.hpp index 92fd4ff..cef35bd 100644 --- a/include/psi/vm/flags/mapping.hpp +++ b/include/psi/vm/flags/mapping.hpp @@ -36,7 +36,6 @@ namespace flags /// //////////////////////////////////////////////////////////////////////////////// - struct mapping { enum struct share_mode diff --git a/include/psi/vm/flags/mapping.win32.hpp b/include/psi/vm/flags/mapping.win32.hpp index cedec9b..88da836 100644 --- a/include/psi/vm/flags/mapping.win32.hpp +++ b/include/psi/vm/flags/mapping.win32.hpp @@ -72,7 +72,7 @@ struct [[ clang::trivial_abi ]] viewing namespace detail { - flags_t BOOST_CC_REG object_access_to_page_access( access_privileges::object, viewing::share_mode ); + flags_t object_access_to_page_access( access_privileges::object, viewing::share_mode ) noexcept; } // namespace detail diff --git a/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp b/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp index 489fc24..4c21a7e 100644 --- a/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp +++ b/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp @@ -69,7 +69,7 @@ namespace detail std::memcpy( prefixed_name + shm_prefix.size(), name, name_length + 1 ); } - BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_WARN_UNUSED_RESULT ) + BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS ) [[ nodiscard ]] file_handle::reference BOOST_CC_REG shm_open ( char const * const name, diff --git a/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp b/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp index 88838fb..5e6b8ed 100644 --- a/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp +++ b/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp @@ -339,8 +339,7 @@ class native_named_memory // { public: static - fallible_result - BOOST_CC_REG create + fallible_result create ( char const * const name, std::size_t const size, @@ -350,16 +349,7 @@ class native_named_memory // return detail::create_mapping_impl::do_map( file_handle::reference{ file_handle::traits::invalid_value }, flags, size, name ); } - fallible_result size() const noexcept - { - auto const p_view( ::MapViewOfFile( get(), 0, 0, 0, 0 ) ); - if ( BOOST_UNLIKELY( !p_view ) ) return error(); - MEMORY_BASIC_INFORMATION info; - BOOST_VERIFY( ::VirtualQuery( p_view, &info, sizeof( info ) ) == sizeof( info ) ); - BOOST_VERIFY( ::UnmapViewOfFile( p_view ) ); - BOOST_ASSUME( info.RegionSize % page_size == 0 ); - return info.RegionSize; - } + auto size() const noexcept { return get_size( *this ); } private: /// \note Required to enable the emplacement constructors of err diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index 20b2301..2dcb7c9 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -33,16 +33,16 @@ #include #include //------------------------------------------------------------------------------ -namespace psi -{ -//------------------------------------------------------------------------------ -namespace vm +namespace psi::vm { //------------------------------------------------------------------------------ namespace detail { [[ noreturn ]] inline void throw_out_of_range() { throw std::out_of_range( "vm::vector access out of bounds" ); } + + template struct size { T value; }; + template struct size {}; } // namespace detail class contiguous_container_storage_base @@ -52,7 +52,7 @@ class contiguous_container_storage_base contiguous_container_storage_base & operator=( contiguous_container_storage_base && ) = default; auto data() noexcept { BOOST_ASSERT_MSG( mapping_, "Paging file not attached" ); return std::assume_aligned( view_.data() ); } - auto data() const noexcept { return const_cast< contiguous_container_storage_base & >( *this ).data(); } + auto data() const noexcept { return const_cast( *this ).data(); } void unmap() noexcept { view_.unmap(); } @@ -155,6 +155,11 @@ class contiguous_container_storage_base std::move( file ), ap::object{ ap::readwrite }, ap::child_process::does_not_inherit, +# ifdef __linux__ + // TODO solve in a cleaner/'in a single place' way + // https://bugzilla.kernel.org/show_bug.cgi?id=8691 mremap: Wrong behaviour expanding a MAP_SHARED anonymous mapping + !file ? flags::share_mode::hidden : +# endif flags::share_mode::shared, mapping::supports_zero_sized_mappings ? mapping_size @@ -176,11 +181,11 @@ class contiguous_container_storage_base mapping mapping_; }; // contiguous_container_storage_base -template < typename sz_t, bool headerless > +template class contiguous_container_storage : public contiguous_container_storage_base, - private std::conditional_t> + private detail::size // checkout a revision prior to March the 21st 2024 for a version that used statically sized header sizes // this approach is more versatile while the overhead should be less than negligible { @@ -189,16 +194,17 @@ class contiguous_container_storage private: static constexpr auto size_size{ headerless ? 0 : sizeof( size_type ) }; + using size_holder = detail::size; public: contiguous_container_storage( ) noexcept requires( headerless ) = default; - explicit contiguous_container_storage( size_type const header_size ) noexcept requires( !headerless ) : std::tuple{ header_size } {} + explicit contiguous_container_storage( size_type const header_size ) noexcept requires( !headerless ) : size_holder{ header_size } {} static constexpr std::uint8_t header_size() noexcept requires( headerless ) { return 0; } [[ gnu::pure ]] size_type header_size() const noexcept requires( !headerless ) { - auto const sz{ std::get<0>( *this ) }; + auto const sz{ size_holder::value }; BOOST_ASSUME( sz >= sizeof( sz_t ) ); return sz; } @@ -329,7 +335,7 @@ class contiguous_container_storage [[ gnu::pure, nodiscard ]] size_type & stored_size() noexcept requires( !headerless ) { auto const p_size{ contiguous_container_storage_base::data() + header_size() - size_size }; - BOOST_ASSERT( reinterpret_cast( p_size ) % alignof( size_type ) == 0 ); + BOOST_ASSERT( reinterpret_cast( p_size ) % alignof( size_type ) == 0 ); return *reinterpret_cast( p_size ); } [[ gnu::pure, nodiscard ]] size_type stored_size() const noexcept requires( !headerless ) @@ -344,7 +350,7 @@ bool constexpr is_trivially_moveable #ifdef __clang__ __is_trivially_relocatable( T ) || #endif - std::is_trivially_move_constructible_v< T > + std::is_trivially_move_constructible_v }; struct default_init_t{}; inline constexpr default_init_t default_init; @@ -352,12 +358,14 @@ struct value_init_t {}; inline constexpr value_init_t value_init ; struct header_info { + using align_t = std::uint16_t; // fit page_size + constexpr header_info() = default; - constexpr header_info( std::uint32_t const size, std::uint8_t const alignment ) noexcept : header_size{ size }, data_extra_alignment{ alignment } {} + constexpr header_info( std::uint32_t const size, align_t const alignment ) noexcept : header_size{ size }, data_extra_alignment{ alignment } {} template - constexpr header_info( std::in_place_type_t, std::uint8_t const extra_alignment = 1 ) noexcept : header_info{ sizeof( T ), extra_alignment } {} + constexpr header_info( std::in_place_type_t, align_t const extra_alignment = 1 ) noexcept : header_info{ sizeof( T ), extra_alignment } {} template - static constexpr header_info make( std::uint8_t const extra_alignment = 1 ) noexcept { return { sizeof( T ), extra_alignment }; } + static constexpr header_info make( align_t const extra_alignment = 1 ) noexcept { return { sizeof( T ), extra_alignment }; } template header_info add_header() const noexcept // support chained headers (class hierarchies) @@ -374,13 +382,13 @@ struct header_info header_info with_alignment_for() const noexcept { BOOST_ASSUME( data_extra_alignment >= 1 ); - return { this->header_size, std::max({ this->data_extra_alignment, alignof( T )... }) }; + return { this->header_size, std::max({ this->data_extra_alignment, alignof( T )... }) }; } std::uint32_t final_header_size() const noexcept { return align_up( header_size, data_extra_alignment ); } std::uint32_t header_size { 0 }; - std::uint8_t data_extra_alignment{ 1 }; // e.g. for vectorization or overlaying complex types over std::byte storage + align_t data_extra_alignment{ 1 }; // e.g. for vectorization or overlaying complex types over std::byte storage }; // header_info // Standard-vector-like/compatible _presistent_ container class template @@ -388,8 +396,8 @@ struct header_info // to trivial_abi types. // Used the Boost.Container vector as a starting skeleton. -template < typename T, typename sz_t = std::size_t, bool headerless_param = true > -requires is_trivially_moveable< T > +template +requires is_trivially_moveable class vector { private: @@ -407,7 +415,7 @@ class vector using const_pointer = T const *; using reference = T &; using const_reference = T const &; - using param_const_ref = std::conditional_t< std::is_trivial_v< T > && ( sizeof( T ) <= 2 * sizeof( void * ) ), T, const_reference >; + using param_const_ref = std::conditional_t< std::is_trivial_v && ( sizeof( T ) <= 2 * sizeof( void * ) ), T, const_reference >; using size_type = sz_t; using difference_type = std::make_signed_t; using iterator = typename span_t:: iterator; @@ -574,7 +582,7 @@ class vector //! Throws: Nothing. //! //! Complexity: Constant. - [[ nodiscard, gnu::pure ]] bool empty() const noexcept { return span().empty(); } + [[ nodiscard, gnu::pure ]] bool empty() const noexcept { return !storage_ || span().empty(); } //! Effects: Returns the number of the elements contained in the vector. //! @@ -588,7 +596,7 @@ class vector //! Throws: Nothing. //! //! Complexity: Constant. - [[ nodiscard ]] static constexpr size_type max_size() noexcept { return static_cast< size_type >( std::numeric_limits< size_type >::max() / sizeof( value_type ) ); } + [[ nodiscard ]] static constexpr size_type max_size() noexcept { return static_cast( std::numeric_limits< size_type >::max() / sizeof( value_type ) ); } //! Effects: Inserts or erases elements at the end such that //! the size becomes n. New elements can be default or value initialized. @@ -596,7 +604,7 @@ class vector //! Throws: If memory allocation throws, or T's copy/move or value initialization throws. //! //! Complexity: Linear to the difference between size() and new_size. - void resize( size_type const new_size, default_init_t ) requires( std::is_trivial_v< T > ) + void resize( size_type const new_size, default_init_t ) requires( std::is_trivial_v ) { storage_.resize( to_byte_sz( new_size ) ); } @@ -830,7 +838,7 @@ class vector //! //! Complexity: Constant. [[ nodiscard ]] T * data() noexcept { return to_t_ptr( storage_.data() ); } - [[ nodiscard ]] T const * data() const noexcept { return const_cast< vector & >( *this ).data(); } + [[ nodiscard ]] T const * data() const noexcept { return const_cast( *this ).data(); } [[ nodiscard ]] span_t span() noexcept { return { data(), size() }; } [[ nodiscard ]] const_span_t span() const noexcept { return { data(), size() }; } @@ -854,10 +862,11 @@ class vector reference emplace_back( Args &&...args ) { storage_.expand( to_byte_sz( size() + 1 ) ); + auto & placeholder{ back() }; if constexpr ( sizeof...( args ) ) - return *std::construct_at( &back(), std::forward< Args >( args )... ); + return *std::construct_at( &placeholder, std::forward( args )... ); else - return *new ( &back() ) T; // default init + return *new ( &placeholder ) T; // default init } //! Effects: Inserts an object of type T constructed with @@ -904,7 +913,7 @@ class vector //! T's copy/move constructor throws. //! //! Complexity: Amortized constant time. - void push_back( T && x ) requires( !std::is_trivial_v< T > ) { emplace_back( std::move( x ) ); } + void push_back( T && x ) requires( !std::is_trivial_v ) { emplace_back( std::move( x ) ); } //! Requires: position must be a valid iterator of *this. //! @@ -924,7 +933,7 @@ class vector //! //! Complexity: If position is end(), amortized constant time //! Linear time otherwise. - iterator insert( const_iterator const position, T && x ) requires( !std::is_trivial_v< T > ) { return emplace( position, std::move( x ) ); } + iterator insert( const_iterator const position, T && x ) requires( !std::is_trivial_v ) { return emplace( position, std::move( x ) ); } //! Requires: position must be a valid iterator of *this. //! @@ -1038,7 +1047,7 @@ class vector //! Throws: Nothing. //! //! Complexity: Constant. - void swap( vector & x ) noexcept { swap( this->storage, x.storage ); } + void swap( vector & x ) noexcept { using std::swap; swap( this->storage_, x.storage_ ); } //! Effects: Erases all the elements of the vector. //! @@ -1072,7 +1081,16 @@ class vector /////////////////////////////////////////////////////////////////////////// auto map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_file ( file, policy ); } - auto map_memory( size_type const size ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_memory( to_byte_sz( size ) ); } + template + auto map_memory( size_type const initial_size = 0, InitPolicy init_policy = {} ) noexcept + { + BOOST_ASSERT( !has_attached_storage() ); + auto result{ storage_.map_memory( to_byte_sz( initial_size ) ) }; + if ( std::is_same_v && initial_size && std::move( result ) ) { + std::uninitialized_default_construct( begin(), end() ); + } + return result; + } bool has_attached_storage() const noexcept { return static_cast( storage_ ); } @@ -1091,7 +1109,7 @@ class vector //! Note: Non-standard extension. bool stable_reserve( size_type new_cap ) = /*TODO*/ delete; - void grow_to( size_type const target_size, default_init_t ) requires( std::is_trivial_v ) + void grow_to( size_type const target_size, default_init_t ) requires( std::is_trivially_default_constructible_v ) { storage_.expand( to_byte_sz( target_size ) ); } @@ -1118,10 +1136,7 @@ class vector } else { - for ( auto & element : span().subspan( current_size ) ) - { - std::construct_at( &element ); - } + std::uninitialized_default_construct( nth( current_size ), end() ); } } PSI_WARNING_DISABLE_POP() @@ -1162,7 +1177,12 @@ class vector PSI_WARNING_GCC_OR_CLANG_DISABLE( -Wsign-conversion ) static T * to_t_ptr ( mapped_view::value_type * const ptr ) noexcept { return reinterpret_cast( ptr ); } static sz_t to_t_sz ( auto const byte_sz ) noexcept { BOOST_ASSUME( byte_sz % sizeof( T ) == 0 ); return static_cast( byte_sz / sizeof( T ) ); } - static sz_t to_byte_sz( auto const sz ) noexcept { return static_cast( sz * sizeof( T ) ); } + static sz_t to_byte_sz( auto const sz ) noexcept + { + auto const rez{ sz * sizeof( T ) }; + BOOST_ASSERT( rez <= std::numeric_limits::max() ); + return static_cast( rez ); + } PSI_WARNING_DISABLE_POP() void shrink_storage_to( size_type const target_size ) noexcept @@ -1178,8 +1198,6 @@ class vector iterator make_space_for_insert( const_iterator const position, size_type const n ) { - using ssize_type = std::make_signed_t; - verify_iterator( position ); auto const position_index{ index_of( position ) }; auto const current_size { size() }; @@ -1202,15 +1220,15 @@ class vector }; // class vector -template < typename T, typename sz_t, bool headerless > +template #if defined( _MSC_VER ) && !defined( NDEBUG ) // Boost.Container flat_set is bugged: tries to dereference end iterators - asserts at runtime with secure/checked STL/containers // https://github.com/boostorg/container/issues/261 -requires is_trivially_moveable< T > -class unchecked_vector : public vector< T, sz_t, headerless > +requires is_trivially_moveable +class unchecked_vector : public vector { public: - using base = vector< T, sz_t, headerless >; + using base = vector; using iterator = typename base:: pointer; using const_iterator = typename base::const_pointer; @@ -1271,14 +1289,12 @@ class unchecked_vector : public vector< T, sz_t, headerless > } }; #else -using unchecked_vector = vector< T, sz_t, headerless >; +using unchecked_vector = vector; #endif -template < typename Vector > -using unchecked = unchecked_vector< typename Vector::value_type, typename Vector::size_type, Vector::headerless >; +template +using unchecked = unchecked_vector; //------------------------------------------------------------------------------ -} // namespace vm -//------------------------------------------------------------------------------ -} // namespace psi +} // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/src/allocation/allocation.impl.hpp b/src/allocation/allocation.impl.hpp index 9bffa3c..50c1ab0 100644 --- a/src/allocation/allocation.impl.hpp +++ b/src/allocation/allocation.impl.hpp @@ -13,26 +13,21 @@ //------------------------------------------------------------------------------ #pragma once -#include +#include //------------------------------------------------------------------------------ -namespace psi -{ -//------------------------------------------------------------------------------ -namespace vm +namespace psi::vm { //------------------------------------------------------------------------------ namespace // internal header { - constexpr void * add( void * const ptr, auto const diff ) noexcept { return static_cast( ptr ) + diff; } - constexpr void const * add( void const * const ptr, auto const diff ) noexcept { return static_cast( ptr ) + diff; } - constexpr auto add( auto const x , auto const diff ) noexcept { return x + diff; } + constexpr void add( void * & ptr, auto const diff ) noexcept { reinterpret_cast( ptr ) += diff; } + constexpr void add( void const * & ptr, auto const diff ) noexcept { reinterpret_cast( ptr ) += diff; } + constexpr void add( auto & x , auto const diff ) noexcept { x += diff; } - constexpr auto sub( auto const x , auto const diff ) noexcept { return x - diff; } + constexpr void sub( auto & x , auto const diff ) noexcept { x -= diff; } } // anonymous namespace //------------------------------------------------------------------------------ -} // namespace vm -//------------------------------------------------------------------------------ -} // namespace psi +} // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/src/allocation/allocation.posix.cpp b/src/allocation/allocation.posix.cpp index 262c66a..ff8736c 100644 --- a/src/allocation/allocation.posix.cpp +++ b/src/allocation/allocation.posix.cpp @@ -12,6 +12,7 @@ //////////////////////////////////////////////////////////////////////////////// //------------------------------------------------------------------------------ #include "allocation.impl.hpp" +#include #include #include diff --git a/src/allocation/allocation.win32.cpp b/src/allocation/allocation.win32.cpp index e80e235..de57382 100644 --- a/src/allocation/allocation.win32.cpp +++ b/src/allocation/allocation.win32.cpp @@ -12,6 +12,7 @@ //////////////////////////////////////////////////////////////////////////////// //------------------------------------------------------------------------------ #include "allocation.impl.hpp" +#include #include #include @@ -24,8 +25,6 @@ namespace psi::vm { //------------------------------------------------------------------------------ -auto const this_process{ reinterpret_cast( static_cast( -1 ) ) }; - enum class deallocation_type : std::uint32_t { free = MEM_RELEASE, @@ -45,7 +44,7 @@ auto alloc( void * & desired_location, std::size_t & size, allocation_type const { NtAllocateVirtualMemory ( - this_process, + nt::current_process, reinterpret_cast< PVOID * >( &desired_location ), 0, // address zero bits &sz, @@ -65,7 +64,7 @@ auto alloc( void * & desired_location, std::size_t & size, allocation_type const } else { - BOOST_ASSUME( ( nt_status == STATUS_NO_MEMORY ) || ( nt_status == STATUS_INVALID_PARAMETER && !size ) ); + BOOST_ASSUME( ( nt_status == NTSTATUS( STATUS_NO_MEMORY ) ) || ( nt_status == NTSTATUS( STATUS_INVALID_PARAMETER ) && !size ) ); desired_location = nullptr; } return nt_status; @@ -86,7 +85,7 @@ void dealloc( void * & address, std::size_t & size, deallocation_type const type #if NATIVE_NT using namespace nt; SIZE_T sz( size ); - auto const result{ NtFreeVirtualMemory( this_process, &address, &sz, std::to_underlying( type ) ) }; + auto const result{ NtFreeVirtualMemory( nt::current_process, &address, &sz, std::to_underlying( type ) ) }; size = static_cast< std::size_t >( sz ); BOOST_ASSUME( result == STATUS_SUCCESS || size == 0 ); #else @@ -94,11 +93,11 @@ void dealloc( void * & address, std::size_t & size, deallocation_type const type #endif } -auto mem_info( void * const pAddress ) noexcept +WIN32_MEMORY_REGION_INFORMATION mem_info( void * const address ) noexcept { - MEMORY_BASIC_INFORMATION info; - BOOST_VERIFY( ::VirtualQueryEx( this_process, pAddress, &info, sizeof( info ) ) == sizeof( info ) ); - BOOST_ASSUME( info.BaseAddress == pAddress ); + WIN32_MEMORY_REGION_INFORMATION info; + BOOST_VERIFY( nt::NtQueryVirtualMemory( nt::current_process, address, nt::MemoryRegionInformation, &info, sizeof( info ), nullptr ) == nt::STATUS_SUCCESS ); + BOOST_ASSUME( info.AllocationBase == address ); return info; } @@ -120,20 +119,18 @@ bool commit( void * const desired_location, std::size_t const size ) noexcept { auto const info{ mem_info( final_address ) }; BOOST_ASSUME( info.AllocationProtect == PAGE_READWRITE ); - BOOST_ASSUME( info.State == MEM_RESERVE ); - BOOST_ASSUME( info.Protect == 0 ); - BOOST_ASSUME( info.Type == MEM_PRIVATE ); - auto regionSize{ std::min( static_cast< std::size_t >( info.RegionSize ), size - final_size ) }; - auto const partial_result{ alloc( final_address, regionSize, allocation_type::commit ) }; + BOOST_ASSUME( info.Private ); + auto region_size{ std::min( static_cast< std::size_t >( info.RegionSize ), size - final_size ) }; + auto const partial_result{ alloc( final_address, region_size, allocation_type::commit ) }; if ( partial_result != STATUS_SUCCESS ) { - BOOST_ASSERT( partial_result == STATUS_NO_MEMORY ); + BOOST_ASSERT( partial_result == NTSTATUS( STATUS_NO_MEMORY ) ); return false; } - BOOST_ASSUME( regionSize <= info.RegionSize ); + BOOST_ASSUME( region_size <= info.RegionSize ); - add( final_address, regionSize ); - add( final_size , regionSize ); + add( final_address, region_size ); + add( final_size , region_size ); } final_address = desired_location; result = STATUS_SUCCESS; @@ -158,11 +155,11 @@ void free( void * address, std::size_t size ) noexcept // emulate support for adjacent/concatenated regions (as supported by mmap) while ( size ) { - std::size_t releasedSize{ 0 }; - dealloc( address, releasedSize, deallocation_type::free ); - BOOST_ASSERT( releasedSize <= size ); - add( address, releasedSize ); - sub( size , releasedSize ); + std::size_t released_size{ 0 }; + dealloc( address, released_size, deallocation_type::free ); + BOOST_ASSUME( released_size <= size ); + add( address, released_size ); + sub( size , released_size ); } } diff --git a/src/allocation/remap.cpp b/src/allocation/remap.cpp index 21cf0b0..2fdcfab 100644 --- a/src/allocation/remap.cpp +++ b/src/allocation/remap.cpp @@ -13,6 +13,7 @@ //------------------------------------------------------------------------------ #include "allocation.impl.hpp" #include +#include #include #include @@ -147,7 +148,7 @@ expand_result expand_back expand_result expand_front ( - std::byte * const address, + std::byte * const address, std::size_t const current_size, std::size_t const required_size, std::size_t const used_capacity, diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp new file mode 100644 index 0000000..7a60074 --- /dev/null +++ b/src/containers/b+tree.cpp @@ -0,0 +1,489 @@ +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +// https://en.wikipedia.org/wiki/B-tree +// https://en.wikipedia.org/wiki/B%2B_tree +// https://opendsa-server.cs.vt.edu/ODSA/Books/CS3/html/BTree.html +// http://www.cburch.com/cs/340/reading/btree/index.html +// https://www.programiz.com/dsa/b-plus-tree +// https://courses.cs.washington.edu/courses/cse332/23su/lectures/9_B_Trees.pdf (version, as this impl, which has different key counts in inner vs leaf nodes) +// https://www.geeksforgeeks.org/b-trees-implementation-in-c +// https://flatcap.github.io/linux-ntfs/ntfs/concepts/tree/index.html +// https://benjamincongdon.me/blog/2021/08/17/B-Trees-More-Than-I-Thought-Id-Want-to-Know +// https://stackoverflow.com/questions/59362113/b-tree-minimum-internal-children-count-explanation +// https://web.archive.org/web/20190126073810/http://supertech.csail.mit.edu/cacheObliviousBTree.html +// https://www.researchgate.net/publication/220225482_Cache-Oblivious_Databases_Limitations_and_Opportunities +// https://www.postgresql.org/docs/current/btree.html +// https://www.scylladb.com/2021/11/23/the-taming-of-the-b-trees +// https://www.scattered-thoughts.net/writing/smolderingly-fast-btrees +// B+tree vs LSM-tree https://www.usenix.org/conference/fast22/presentation/qiao +// Data Structure Visualizations https://www.cs.usfca.edu/~galles/visualization/Algorithms.html +// Griffin: Fast Transactional Database Index with Hash and B+-Tree https://ieeexplore.ieee.org/abstract/document/10678674 +// Restructuring the concurrent B+-tree with non-blocked search operations https://www.sciencedirect.com/science/article/abs/pii/S002002550200261X +// Cache-Friendly Search Trees https://arxiv.org/pdf/1907.01631 +// ZBTree: A Fast and Scalable B+-Tree for Persistent Memory https://ieeexplore.ieee.org/document/10638243 + +// https://abseil.io/about/design/btree +// https://github.com/tlx/tlx/tree/master/tlx/container +// https://github.com/jeffplaisance/BppTree +// https://github.com/postgres/postgres/blob/master/src/backend/access/nbtree/README +// https://github.com/scylladb/scylladb/blob/master/utils/intrusive_btree.hh + +// https://en.wikipedia.org/wiki/Judy_array + +bptree_base::bptree_base( header_info const hdr_info ) noexcept + : + nodes_{ hdr_info.add_header
() } +{} + +void bptree_base::clear() noexcept +{ + nodes_.clear(); + hdr() = {}; +} + +std::span +bptree_base::user_header_data() noexcept { return header_data().second; } + +[[ gnu::pure, gnu::hot, gnu::always_inline ]] +bptree_base::header & +bptree_base::hdr() noexcept { return *header_data().first; } + +bptree_base::storage_result +bptree_base::map_memory( std::uint32_t const initial_capacity_as_number_of_nodes ) noexcept +{ + storage_result success{ nodes_.map_memory( initial_capacity_as_number_of_nodes, value_init ) }; +#ifndef NDEBUG + if ( std::move( success ) ) + { + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); + } +#endif + if ( std::move( success ) ) + { + hdr() = {}; + if ( initial_capacity_as_number_of_nodes ) { + assign_nodes_to_free_pool( 0 ); + } + } + return success; +} + +void bptree_base::reserve_additional( node_slot::value_type additional_nodes ) +{ + auto const preallocated_count{ hdr().free_node_count_ }; + additional_nodes -= std::min( preallocated_count, additional_nodes ); + auto const current_size{ nodes_.size() }; + nodes_.grow_by( additional_nodes, value_init ); +#ifndef NDEBUG + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); +#endif + assign_nodes_to_free_pool( current_size ); +} + +void bptree_base::reserve( node_slot::value_type new_capacity_in_number_of_nodes ) +{ + if ( new_capacity_in_number_of_nodes <= nodes_.capacity() ) + return; + auto const current_size{ nodes_.size() }; + nodes_.grow_to( new_capacity_in_number_of_nodes, value_init ); +#ifndef NDEBUG + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); +#endif + assign_nodes_to_free_pool( current_size ); +} +[[ gnu::cold ]] +void bptree_base::assign_nodes_to_free_pool( node_slot::value_type const starting_node ) noexcept +{ + for ( auto & n : std::views::reverse( std::span{ nodes_ }.subspan( starting_node ) ) ) + free( n ); +} + +void bptree_base::rshift_sibling_parent_pos( node_header & node ) noexcept +{ + auto p_node{ &node }; + while ( p_node->right ) + { + auto & right{ this->node( p_node->right ) }; + BOOST_ASSUME( right.parent_child_idx == p_node->parent_child_idx ); + ++right.parent_child_idx; + p_node = &right; + } +} + +void bptree_base::update_right_sibling_link( node_header const & left_node, node_slot const left_node_slot ) noexcept +{ + BOOST_ASSUME( slot_of( left_node ) == left_node_slot ); + if ( left_node.right ) [[ likely ]] + { + auto & right_back_left{ right( left_node ).left }; + BOOST_ASSUME( right_back_left != left_node_slot ); + right_back_left = left_node_slot; + } +} + +void bptree_base::unlink_and_free_node( node_header & node, node_header & cached_left_sibling ) noexcept +{ + auto & left { cached_left_sibling }; + auto & right{ node }; + BOOST_ASSUME( left .right == slot_of( right ) ); + BOOST_ASSUME( right.left == slot_of( left ) ); + left.right = right.right; + update_right_sibling_link( left, right.left ); + free( node ); +} + +void bptree_base::update_leaf_list_ends( node_header & removed_leaf ) noexcept +{ + auto & first_leaf{ hdr().first_leaf_ }; + auto & last_leaf{ hdr().last_leaf_ }; + auto const slot{ slot_of( removed_leaf ) }; + if ( slot == first_leaf ) [[ unlikely ]] + { + BOOST_ASSUME( !removed_leaf.left ); + first_leaf = removed_leaf.right; + } + if ( slot == last_leaf ) [[ unlikely ]] + { + BOOST_ASSUME( !removed_leaf.right ); + last_leaf = removed_leaf.left; + } +} + +void bptree_base::unlink_and_free_leaf( node_header & leaf, node_header & cached_left_sibling ) noexcept +{ + update_leaf_list_ends( leaf ); + unlink_and_free_node( leaf, cached_left_sibling ); +} + + +void bptree_base::unlink_left( node_header & nd ) noexcept +{ + if ( !nd.left ) + return; + auto & left_link{ left( nd ).right }; + BOOST_ASSUME( left_link == slot_of( nd ) ); + left_link = nd.left = {}; +} + +void bptree_base::unlink_right( node_header & nd ) noexcept +{ + if ( !nd.right ) + return; + auto & right_link{ right( nd ).left }; + BOOST_ASSUME( right_link == slot_of( nd ) ); + right_link = nd.right = {}; +} + +void bptree_base::link( node_header & left, node_header & right ) const noexcept +{ + BOOST_ASSUME( !left .right ); + BOOST_ASSUME( !right.left ); + left .right = slot_of( right ); + right.left = slot_of( left ); +} + +[[ gnu::noinline, gnu::sysv_abi ]] +std::pair +bptree_base::new_spillover_node_for( node_header & existing_node ) +{ + auto const existing_node_slot{ slot_of( existing_node ) }; + auto & right_node { new_node() }; + auto & left_node { node( existing_node_slot ) }; // handle relocation (by new_node()) + auto const right_node_slot { slot_of( right_node ) }; + + // insert into the level dlinked list (TODO extract into small specialized dlinked list functions) + // node <- left/lesser | new_node -> right/greater + right_node.left = existing_node_slot; + right_node.right = left_node.right; + left_node.right = right_node_slot; + update_right_sibling_link( right_node, right_node_slot ); + right_node.parent = left_node.parent; + right_node.parent_child_idx = left_node.parent_child_idx + 1; + //right-sibling parent_child_idx rshifting is performed by insert_into_* + //rshift_sibling_parent_pos( right_node ); + + return std::make_pair( existing_node_slot, right_node_slot ); +} +[[ gnu::noinline ]] +bptree_base::node_placeholder & +bptree_base::new_root( node_slot const left_child, node_slot const right_child ) +{ + auto & new_root{ new_node() }; + auto & hdr { this->hdr() }; + auto & left { node( left_child ) }; BOOST_ASSUME( left.is_root() ); + auto & right{ node( right_child ) }; + new_root.left = new_root.right = {}; + new_root.num_vals = 1; // the minimum of two children with one separator key + hdr.root_ = slot_of( new_root ); + left .parent = hdr.root_; + right.parent = hdr.root_; + BOOST_ASSUME( left .parent_child_idx == 0 ); + BOOST_ASSUME( right.parent_child_idx == 1 ); + ++hdr.depth_; + return new_root; +} + +bptree_base::base_iterator::base_iterator( node_pool & nodes, iter_pos const pos ) noexcept + : + nodes_{}, + pos_{ pos } +{ + update_pool_ptr( nodes ); +} + +void bptree_base::base_iterator::update_pool_ptr( node_pool & nodes ) const noexcept +{ +#ifndef NDEBUG // for bounds checking + nodes_ = nodes; +#else + nodes_ = nodes.data(); +#endif +} + +[[ gnu::pure ]] +bptree_base::node_header & +bptree_base::base_iterator::node() const noexcept { return nodes_[ *pos_.node ]; } + +bptree_base::base_iterator & +bptree_base::base_iterator::operator++() noexcept +{ + auto & node{ this->node() }; + BOOST_ASSERT_MSG( pos_.value_offset < node.num_vals, "Iterator at end: not incrementable" ); + BOOST_ASSUME( node.num_vals >= 1 ); + BOOST_ASSUME( pos_.value_offset < node.num_vals ); + if ( ++pos_.value_offset == node.num_vals ) [[ unlikely ]] + { + // have to perform this additional check to allow an iterator to arrive + // to the end position + if ( node.right ) [[ likely ]] + { + pos_.node = node.right; + pos_.value_offset = 0; + } + } + return *this; +} +[[ clang::no_sanitize( "implicit-conversion" ) ]] +bptree_base::base_iterator & +bptree_base::base_iterator::operator--() noexcept +{ + auto & node{ this->node() }; + BOOST_ASSERT_MSG( ( pos_.value_offset > 0 ) || node.left, "Iterator at end: not incrementable" ); + BOOST_ASSUME( pos_.value_offset <= node.num_vals ); + BOOST_ASSUME( node.num_vals >= 1 ); + if ( pos_.value_offset-- == 0 ) [[ unlikely ]] + { + pos_.node = node.left; + pos_.value_offset = this->node().num_vals - 1; + } + return *this; +} + +bool bptree_base::base_iterator::operator==( base_iterator const & other ) const noexcept +{ + BOOST_ASSERT_MSG( &this->nodes_[ 0 ] == &other.nodes_[ 0 ], "Comparing iterators from different containers" ); + return ( this->pos_.node == other.pos_.node ) && ( this->pos_.value_offset == other.pos_.value_offset ); +} + + +bptree_base::base_random_access_iterator & +bptree_base::base_random_access_iterator::operator+=( difference_type const n ) noexcept +{ + BOOST_ASSERT_MSG( pos_.node, "Iterator at end: not incrementable" ); + + if ( n >= 0 ) + { + auto un{ static_cast( n ) }; + for ( ;; ) + { + auto & node{ this->node() }; + auto const available_offset{ static_cast( node.num_vals - 1 - pos_.value_offset ) }; + if ( available_offset >= un ) { + pos_.value_offset += static_cast( un ); + BOOST_ASSUME( pos_.value_offset < node.num_vals ); + break; + } else { + un -= ( available_offset + 1 ); + // Here we don't have to perform the same check as in the + // fwd_iterator increment since (end) iterator comparison is + // done solely through the index_ member. + pos_.node = node.right; + pos_.value_offset = 0; + if ( un == 0 ) [[ unlikely ]] + break; + BOOST_ASSERT_MSG( pos_.node, "Incrementing out of bounds" ); + } + } + index_ += static_cast( n ); + } + else + { + auto un{ static_cast( -n ) }; + BOOST_ASSERT_MSG( index_ >= un, "Moving iterator out of bounds" ); + for ( ;; ) + { + auto const available_offset{ pos_.value_offset }; + if ( available_offset >= un ) { + pos_.value_offset -= static_cast( un ); + BOOST_ASSUME( pos_.value_offset < node().num_vals ); + break; + } else { + un -= ( available_offset + 1 ); + pos_.node = node().left; + pos_.value_offset = node().num_vals - 1; + BOOST_ASSERT_MSG( pos_.node, "Incrementing out of bounds" ); + BOOST_ASSUME( pos_.value_offset < node().num_vals ); + } + } + index_ -= static_cast( -n ); + } + + BOOST_ASSUME( !pos_.node || ( pos_.value_offset < node().num_vals ) ); + + return *this; +} + + +void bptree_base::swap( bptree_base & other ) noexcept +{ + using std::swap; + swap( this->nodes_, other.nodes_ ); +#ifndef NDEBUG + swap( this->p_hdr_ , other.p_hdr_ ); + swap( this->p_nodes_, other.p_nodes_ ); +#endif +} + +bptree_base::base_iterator bptree_base::make_iter( iter_pos const pos ) noexcept { return { nodes_, pos }; } +bptree_base::base_iterator bptree_base::make_iter( node_slot const node, node_size_type const offset ) noexcept { return make_iter(iter_pos{ node, offset }); } +bptree_base::base_iterator bptree_base::make_iter( node_header const & node, node_size_type const offset ) noexcept { return make_iter( slot_of( node ), offset ); } +bptree_base::base_iterator bptree_base::make_iter( insert_pos_t const next_pos ) noexcept +{ + auto iter{ make_iter( next_pos.node, next_pos.next_insert_offset ) }; + --iter; + return iter; +} + +[[ gnu::pure ]] bptree_base::iter_pos bptree_base::begin_pos() const noexcept { return { this->first_leaf(), 0 }; } +[[ gnu::pure ]] bptree_base::iter_pos bptree_base:: end_pos() const noexcept { + auto const last_leaf{ hdr().last_leaf_ }; + return { last_leaf, last_leaf ? node( last_leaf ).num_vals : node_size_type{} }; +} + +[[ gnu::pure ]] bptree_base::base_iterator bptree_base::begin() noexcept { return make_iter( begin_pos() ); } +[[ gnu::pure ]] bptree_base::base_iterator bptree_base::end () noexcept { return make_iter( end_pos() ); } + +[[ gnu::pure ]] bptree_base::base_random_access_iterator bptree_base::ra_begin() noexcept { return { *this, begin_pos(), 0 }; } +[[ gnu::pure ]] bptree_base::base_random_access_iterator bptree_base::ra_end () noexcept { return { *this, end_pos(), size() }; } + +[[ gnu::cold ]] +bptree_base::node_header & +bptree_base::create_root() +{ + BOOST_ASSUME( !hdr().root_ ); + BOOST_ASSUME( !hdr().depth_ ); + BOOST_ASSUME( !hdr().size_ ); + // the initial/'lone' root node is a leaf node + auto & root{ new_node() }; + auto & hdr { this->hdr() }; + BOOST_ASSUME( root.num_vals == 0 ); + root.num_vals = 1; + root.left = {}; + root.right = {}; + hdr.root_ = slot_of( root ); + hdr.first_leaf_ = hdr.root_; + hdr.last_leaf_ = hdr.root_; + hdr.depth_ = 1; + hdr.size_ = 1; + return root; +} + +bptree_base::depth_t bptree_base:: leaf_level() const noexcept { BOOST_ASSUME( hdr().depth_ ); return hdr().depth_ - 1; } + bool bptree_base::is_leaf_level( depth_t const level ) const noexcept { return level == leaf_level(); } + +bool bptree_base::is_my_node( node_header const & node ) const noexcept +{ + return ( &node >= &nodes_.front() ) && ( &node <= &nodes_.back() ); +} + +bptree_base::node_slot +bptree_base::slot_of( node_header const & node ) const noexcept +{ + BOOST_ASSUME( is_my_node( node ) ); + return { static_cast( static_cast( &node ) - nodes_.data() ) }; +} + + +[[ gnu::noinline ]] +bptree_base::node_placeholder & +bptree_base::new_node() +{ + auto & hdr { this->hdr() }; + auto & free_list{ hdr.free_list_ }; + if ( free_list ) + { + BOOST_ASSUME( hdr.free_node_count_ ); + auto & cached_node{ node( free_list ) }; + BOOST_ASSUME( !cached_node.num_vals ); + BOOST_ASSUME( !cached_node.left ); + free_list = cached_node.right; + unlink_right( cached_node ); + --hdr.free_node_count_; + return as( cached_node ); + } + auto & new_node{ nodes_.emplace_back() }; + BOOST_ASSUME( !new_node.num_vals ); + BOOST_ASSUME( !new_node.left ); + BOOST_ASSUME( !new_node.right ); +#ifndef NDEBUG + p_hdr_ = &this->hdr(); + p_nodes_ = nodes_.data(); +#endif + return new_node; +} + +void bptree_base::free( node_header & node ) noexcept +{ + auto & hdr{ this->hdr() }; + auto & free_list{ hdr.free_list_ }; + auto & free_node{ static_cast( node ) }; + auto const free_node_slot{ slot_of( free_node ) }; + BOOST_ASSUME( free_node_slot != hdr.last_leaf_ ); // should have been handled in the dedicated overload +#ifndef NDEBUG + if ( free_node.left ) + BOOST_ASSERT( left( free_node ).right != free_node_slot ); + if ( free_node.right ) + BOOST_ASSERT( right( free_node ).left != free_node_slot ); +#endif + // TODO (re)consider whether the calling code is expected to reset num_vals + // (and thus mark the node as 'free'/empty) and/or unlink from its siblings + // and the parent (which complicates code but possibly catches errors + // earlier) or have free perform all of the cleanup. + //BOOST_ASSUME( node.num_vals == 0 ); + // We have to touch the header (i.e. its residing cache line) anyway here + // (to update the right link) so reset/setup the whole header right now for + // the new allocation step. + static_cast( free_node ) = {}; + // update the right link + if ( free_list ) { BOOST_ASSUME( hdr.free_node_count_ ); link( free_node, this->node( free_list ) ); } + else { BOOST_ASSUME( !hdr.free_node_count_ ); } + free_list = free_node_slot; + ++hdr.free_node_count_; +} + +void bptree_base::free_leaf( node_header & leaf ) noexcept +{ + update_leaf_list_ends( leaf ); + free( leaf ); +} + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/src/detail/nt.cpp b/src/detail/nt.cpp index bc691e9..fbfea22 100644 --- a/src/detail/nt.cpp +++ b/src/detail/nt.cpp @@ -21,6 +21,8 @@ //------------------------------------------------------------------------------ #include +#include + #include #pragma comment( lib, "ntdll.lib" ) @@ -37,8 +39,11 @@ namespace detail [[ gnu::constructor( 101 ) ]] void init_ntdll_handle() noexcept { ntdll = ::GetModuleHandleW( L"ntdll.dll" ); } #else + PSI_WARNING_DISABLE_PUSH() + PSI_WARNING_MSVC_DISABLE( 4073 ) // initializers put in library initialization area #pragma init_seg( lib ) HMODULE const ntdll{ ::GetModuleHandleW( L"ntdll.dll" ) }; + PSI_WARNING_DISABLE_POP() #endif BOOST_ATTRIBUTES( BOOST_COLD, BOOST_RESTRICTED_FUNCTION_L3, BOOST_RESTRICTED_FUNCTION_RETURN ) diff --git a/src/flags/mapping.win32.cpp b/src/flags/mapping.win32.cpp index d8e9951..6c42e40 100644 --- a/src/flags/mapping.win32.cpp +++ b/src/flags/mapping.win32.cpp @@ -68,7 +68,7 @@ bool viewing::is_cow() const noexcept namespace detail { - flags_t BOOST_CC_REG object_access_to_page_access( access_privileges::object const object_access, viewing::share_mode const share_mode ) + flags_t object_access_to_page_access( access_privileges::object const object_access, viewing::share_mode const share_mode ) noexcept { // Generate CreateFileMapping flags from access_privileges::object/MapViewOfFile flags using access_rights = access_privileges; diff --git a/src/mappable_objects/file/file.posix.cpp b/src/mappable_objects/file/file.posix.cpp index d0e2ef5..a04c4ce 100644 --- a/src/mappable_objects/file/file.posix.cpp +++ b/src/mappable_objects/file/file.posix.cpp @@ -110,7 +110,7 @@ mapping create_mapping # if defined( MAP_SHARED_VALIDATE ) // mmap fails with EINVAL (under WSL kernel 5.15 w/ ArchLinux) when // MAP_SHARED_VALIDATE is combined with MAP_ANONYMOUS - if ( ( std::to_underlying( flags::mapping::share_mode::shared ) == MAP_SHARED_VALIDATE ) && ( view_flags.flags & MAP_SHARED_VALIDATE ) ) + if ( ( view_flags.flags & MAP_TYPE ) == MAP_SHARED_VALIDATE ) { view_flags.flags &= ~MAP_SHARED_VALIDATE; view_flags.flags |= MAP_SHARED; diff --git a/src/mappable_objects/file/file.win32.cpp b/src/mappable_objects/file/file.win32.cpp index e53feb4..605e6aa 100644 --- a/src/mappable_objects/file/file.win32.cpp +++ b/src/mappable_objects/file/file.win32.cpp @@ -117,6 +117,16 @@ namespace detail { HANDLE handle{ handle_traits::invalid_value }; LARGE_INTEGER maximum_size{ .QuadPart = static_cast( size ) }; + BOOST_ASSERT_MSG( std::uint64_t( maximum_size.QuadPart ) == size, "Unsupported section size" ); + if ( !file ) + { + // Windows (11 23H2) does not (still) seem to support resizing of pagefile-backed mappings + // so we emulate those by creating a 2GB one and counting on: + // - the NT kernel to be lazy and + // - that amount to be 'enough for everyone'. + maximum_size.QuadPart = std::numeric_limits::max(); + } + auto const nt_result { nt::NtCreateSection // TODO use it for named sections also diff --git a/src/mappable_objects/shared_memory/mem.posix.cpp b/src/mappable_objects/shared_memory/mem.posix.cpp index 772f92e..b2dd85c 100644 --- a/src/mappable_objects/shared_memory/mem.posix.cpp +++ b/src/mappable_objects/shared_memory/mem.posix.cpp @@ -234,7 +234,7 @@ namespace detail return result; } - BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_WARN_UNUSED_RESULT ) + [[ nodiscard ]] BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS ) bool named_semaphore::semop( short const opcode, bool const nowait /*= false*/ ) noexcept { // http://linux.die.net/man/2/semop diff --git a/src/mapped_view/mapped_view.cpp b/src/mapped_view/mapped_view.cpp index e25fb76..a12b6ff 100644 --- a/src/mapped_view/mapped_view.cpp +++ b/src/mapped_view/mapped_view.cpp @@ -13,6 +13,8 @@ //------------------------------------------------------------------------------ #include #include + +#include // memcpy //------------------------------------------------------------------------------ namespace psi::vm { @@ -89,9 +91,11 @@ template fallible_result basic_mapped_view::expand( std::size_t const target_size, mapping & original_mapping ) noexcept { - auto const current_address { const_cast( this->data() ) }; - auto const current_size { this->size() }; - auto const kernel_current_size{ align_up( current_size, commit_granularity ) }; + // TODO kill duplication with remap.cpp::expand() + + auto const current_address { const_cast( this->data() ) }; + auto const current_size { this->size() }; + auto const kernel_current_size{ align_up( current_size, commit_granularity ) }; if ( kernel_current_size >= target_size ) { BOOST_ASSUME( current_address ); @@ -101,6 +105,31 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o } #if defined( __linux__ ) + // unlike realloc mremap does not support functioning (also) as 'malloc' + // i.e. it does not support nullptr as the 'old address' + if ( !current_address ) [[ unlikely ]] // initial mapping + { + BOOST_ASSUME( !current_size ); + auto initial_address + { + posix::mmap + ( + nullptr, + target_size, + original_mapping.view_mapping_flags.protection, + original_mapping.view_mapping_flags.flags, + original_mapping.get(), + 0 + ) + }; + if ( initial_address ) [[ likely ]] + { + static_cast( *this ) = { static_cast( initial_address ), target_size }; + return err::success; + } + return error_t{}; + } + if ( auto const new_address{ ::mremap( current_address, current_size, target_size, std::to_underlying( reallocation_type::moveable ) ) }; @@ -110,12 +139,27 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o static_cast( *this ) = { static_cast( new_address ), target_size }; return err::success; } -#endif // linux mremap + return error_t{}; +#else auto const current_offset{ 0U }; // TODO: what if this is an offset view? auto const target_offset { current_offset + kernel_current_size }; auto const additional_tail_size{ target_size - kernel_current_size }; auto const tail_target_address { current_address + kernel_current_size }; -#ifndef _WIN32 +#ifdef _WIN32 + ULARGE_INTEGER const win32_offset{ .QuadPart = target_offset }; + auto const new_address + { + ::MapViewOfFileEx + ( + original_mapping.get(), + original_mapping.view_mapping_flags.map_view_flags, + win32_offset.HighPart, + win32_offset.LowPart, + additional_tail_size, + tail_target_address + ) + }; +#else auto new_address { posix::mmap @@ -131,34 +175,20 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o if ( ( new_address != tail_target_address ) && // On POSIX the target address is only a hint (while MAP_FIXED may overwrite existing mappings) - ( current_address != nullptr ) // in case of starting with an empty/null/zero-sized views (is it worth the special handling?) + ( current_address != nullptr ) // in case of starting with an empty/null/zero-sized view (is it worth the special handling?) ) { BOOST_VERIFY( ::munmap( new_address, additional_tail_size ) == 0 ); new_address = nullptr; } -#else - ULARGE_INTEGER const win32_offset{ .QuadPart = target_offset }; - auto const new_address - { // TODO this (appending) requires complexity to be added on the unmap side (see mapper::unmap) - ::MapViewOfFileEx - ( - original_mapping.get(), - original_mapping.view_mapping_flags.map_view_flags, - win32_offset.HighPart, - win32_offset.LowPart, - additional_tail_size, - tail_target_address - ) - }; #endif if ( new_address ) [[ likely ]] { if ( current_address != nullptr ) [[ likely ]] { BOOST_ASSUME( new_address == current_address + kernel_current_size ); -# ifdef __linux__ - BOOST_ASSERT_MSG( false, "mremap failed but an overlapping mmap succeeded!?" ); // behaviour investigation +# ifdef __linux__ // no longer exercised path (i.e. linux only relies on mremap) + BOOST_ASSERT_MSG( false, "mremap failed but an adjacent mmap succeeded!?" ); // behaviour investigation # endif static_cast( *this ) = { current_address, target_size }; } @@ -172,8 +202,25 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o auto remapped_span{ this->map( original_mapping, 0, target_size )() }; if ( !remapped_span ) return remapped_span.error(); +#ifndef _WIN32 + // Linux handles relocation-on-expansion implicitly with mremap (it even + // supports creating new mappings of the same pages/physical memory by + // specifying zero for old_size -> TODO use this to implement 'multimappable' + // mappings, 'equivalents' of NtCreateSection instead of going through shm), + // Windows tracks the source data/backing storage through the mapping + // handle. For others we have to the the new-copy-free old dance - but this + // is a quick-fix as this still does not support 'multiply remappable + // mapping' semantics a la NtCreateSection - TODO: shm_mkstemp, SHM_ANON, + // memfd_create... + // https://github.com/lassik/shm_open_anon + if ( !original_mapping.is_file_based() ) + std::memcpy( const_cast( remapped_span->data() ), this->data(), this->size() ); + else +#endif + BOOST_ASSERT_MSG( std::memcmp( remapped_span->data(), this->data(), this->size() ) == 0, "View expansion garbled data." ); *this = std::move( *remapped_span ); return err::success; +#endif // end of 'no linux mremap' implementation } // view::expand() template class basic_mapped_view; diff --git a/src/mapped_view/mapped_view.win32.cpp b/src/mapped_view/mapped_view.win32.cpp index cabae35..1c83636 100644 --- a/src/mapped_view/mapped_view.win32.cpp +++ b/src/mapped_view/mapped_view.win32.cpp @@ -14,6 +14,7 @@ /// //////////////////////////////////////////////////////////////////////////////// //------------------------------------------------------------------------------ +#include "../allocation/allocation.impl.hpp" #include #include #include @@ -98,29 +99,34 @@ map #endif } +// defined in allocation.win32.cpp +WIN32_MEMORY_REGION_INFORMATION mem_info( void * const ) noexcept; + namespace { - [[ maybe_unused ]] - auto mem_info( void * const address ) noexcept + BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_RESTRICTED_FUNCTION_L1 ) BOOST_NOINLINE + void unmap_concatenated( void * address, std::size_t size ) noexcept { - MEMORY_BASIC_INFORMATION info; - BOOST_VERIFY( ::VirtualQueryEx( nt::current_process, address, &info, sizeof( info ) ) == sizeof( info ) ); - return info; + // emulate support for adjacent/concatenated regions (as supported by mmap) + // (duplicated logic from free() in allocaiton.win32.cpp) + for ( ;; ) + { + auto const region_size{ mem_info( address ).RegionSize }; + BOOST_ASSUME( region_size <= align_up( size, reserve_granularity ) ); + BOOST_VERIFY( ::UnmapViewOfFile( address ) ); + if ( region_size >= size ) + break; + add( address, region_size ); + sub( size , region_size ); + } } } // anonymous namespace -BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_RESTRICTED_FUNCTION_L1 ) +BOOST_ATTRIBUTES( BOOST_EXCEPTIONLESS, BOOST_RESTRICTED_FUNCTION_L1 ) void unmap( mapped_span const view ) noexcept { - BOOST_ASSERT_MSG - ( // greater-or-equal because of the inability of Windows to do a proper unmap_partial - // (so we do discard() calls and shrinking of the view-span only, while the actual - // region remains in its original size) - mem_info( view.data() ).RegionSize >= align_up( view.size(), commit_granularity ) || view.empty(), - "TODO: implement a loop to handle concatenated ranges" - // see mapped_view::expand, Windows does not allow partial unmaps or merging of mapped views/ranges - ); - BOOST_VERIFY( ::UnmapViewOfFile( view.data() ) || view.empty() ); + if ( !view.empty() ) + unmap_concatenated( view.data(), view.size() ); } void unmap_partial( mapped_span const range ) noexcept diff --git a/src/mapping/mapping.win32.cpp b/src/mapping/mapping.win32.cpp index 34ba362..c3bc16c 100644 --- a/src/mapping/mapping.win32.cpp +++ b/src/mapping/mapping.win32.cpp @@ -34,33 +34,42 @@ std::uint64_t get_size( mapping::const_handle const mapping_handle ) noexcept using namespace nt; SECTION_BASIC_INFORMATION info; auto const result{ NtQuerySection( mapping_handle.value, SECTION_INFORMATION_CLASS::SectionBasicInformation, &info, sizeof( info ), nullptr ) }; - BOOST_VERIFY( NT_SUCCESS( result ) ); + BOOST_ASSUME( result == STATUS_SUCCESS ); return info.SectionSize.QuadPart; } namespace detail::create_mapping_impl { HANDLE map_file( file_handle::reference file, flags::flags_t flags, std::uint64_t size ) noexcept; } -err::fallible_result set_size( mapping & __restrict the_mapping, std::uint64_t const new_size ) noexcept +err::fallible_result set_size( mapping & the_mapping, std::uint64_t const new_size ) noexcept { using namespace nt; - LARGE_INTEGER ntsz{ .QuadPart = static_cast( new_size ) }; - auto const result{ NtExtendSection( the_mapping.get(), &ntsz ) }; - if ( !NT_SUCCESS( result ) ) [[ unlikely ]] - return result; - BOOST_ASSERT( ntsz.QuadPart >= static_cast( new_size ) ); - if ( ntsz.QuadPart > static_cast( new_size ) ) + if ( the_mapping.is_file_based() ) { - // NtExtendSection does not support downsizing - use it also as a size getter (avoid get_size call) - BOOST_ASSERT( ntsz.QuadPart == static_cast( get_size( the_mapping ) ) ); - the_mapping.close(); // no strong guarantee :/ - auto const file_reisze_result{ set_size( the_mapping.underlying_file(), new_size )() }; - if ( !file_reisze_result ) - return nt::error/*...mrmlj...*/( file_reisze_result.error().get() ); // TODO fully move to NativeNT API https://cpp.hotexamples.com/examples/-/-/NtSetInformationFile/cpp-ntsetinformationfile-function-examples.html - auto const new_mapping_handle{ detail::create_mapping_impl::map_file( the_mapping.file, the_mapping.create_mapping_flags, new_size ) }; - the_mapping.reset( new_mapping_handle ); - if ( !the_mapping ) - return nt::error/*...mrmlj...*/( err::last_win32_error::get() ); + LARGE_INTEGER ntsz{ .QuadPart = static_cast( new_size ) }; + auto const result{ NtExtendSection( the_mapping.get(), &ntsz ) }; + if ( !NT_SUCCESS( result ) ) [[ unlikely ]] + return result; + + BOOST_ASSERT( ntsz.QuadPart >= static_cast( new_size ) ); + if ( ntsz.QuadPart > static_cast( new_size ) ) + { + // NtExtendSection does not support downsizing - use it also as a size getter (avoid get_size call) + BOOST_ASSERT( ntsz.QuadPart == static_cast( get_size( the_mapping ) ) ); + the_mapping.close(); // no strong guarantee :/ + auto const file_reisze_result{ set_size( the_mapping.underlying_file(), new_size )() }; + if ( !file_reisze_result ) + return nt::error/*...mrmlj...*/( file_reisze_result.error().get() ); // TODO fully move to NativeNT API https://cpp.hotexamples.com/examples/-/-/NtSetInformationFile/cpp-ntsetinformationfile-function-examples.html + auto const new_mapping_handle{ detail::create_mapping_impl::map_file( the_mapping.file, the_mapping.create_mapping_flags, new_size ) }; + the_mapping.reset( new_mapping_handle ); + if ( !the_mapping ) + return nt::error/*...mrmlj...*/( err::last_win32_error::get() ); + } + } + else + { + if ( new_size > get_size( the_mapping ) ) [[ unlikely ]] + return nt::STATUS_SECTION_NOT_EXTENDED; } return err::success; } diff --git a/src/pch.hpp b/src/pch.hpp new file mode 100644 index 0000000..ad07223 --- /dev/null +++ b/src/pch.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#undef ERROR +#endif \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8f996c7..af92336 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,16 +1,20 @@ # psi::vm tests # Fix ODR violations and compiler and linker option mismatches: add GTest after everything is already setup. -CPMAddPackage( "gh:google/googletest@1.14.0" ) +CPMAddPackage( "gh:google/googletest@1.15.2" ) include( GoogleTest ) set( vm_test_sources + "${CMAKE_CURRENT_SOURCE_DIR}/b+tree.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/vector.cpp" ) add_executable( vm_unit_tests EXCLUDE_FROM_ALL ${vm_test_sources} ) target_link_libraries( vm_unit_tests PRIVATE GTest::gtest_main psi::vm ) +if ( NOT MSVC OR CLANG_CL ) #:wat: msvc fails to find the pch file in release builds!? + target_precompile_headers( vm_unit_tests REUSE_FROM psi_vm ) +endif() set_target_properties( vm_unit_tests diff --git a/test/b+tree.cpp b/test/b+tree.cpp new file mode 100644 index 0000000..9d8c923 --- /dev/null +++ b/test/b+tree.cpp @@ -0,0 +1,232 @@ +#include +#include + +#include +#include + +#define HAVE_ABSL 0 +#if HAVE_ABSL +#include +#endif + +#include + +#include +#include +#include +#include +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +#ifdef NDEBUG // bench only release builds + +namespace +{ + using timer = std::chrono::high_resolution_clock; + using duration = std::chrono::nanoseconds; + + duration time_insertion( auto & container, auto const & data ) + { + auto const start{ timer::now() }; + container.insert( data.begin(), data.end() ); + return std::chrono::duration_cast( timer::now() - start ) / data.size(); + } + duration time_lookup( auto const & container, auto const & data ) noexcept + { + auto const start{ timer::now() }; + for ( auto const x : data ) { + EXPECT_EQ( *container.find( x ), x ); + } + return std::chrono::duration_cast( timer::now() - start ) / data.size(); + } +} // anonymous namespace + +TEST( bp_tree, benchamrk ) +{ + auto const test_size{ 7654321 }; + auto const seed{ std::random_device{}() }; + std::mt19937 rng{ seed }; + + std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; + auto numbers{ std::ranges::to( sorted_numbers ) }; + std::ranges::shuffle( numbers, rng ); + + psi::vm ::bp_tree bpt; bpt.map_memory(); + boost::container::flat_set flat_set; +#if HAVE_ABSL + absl ::btree_set abpt; +#endif + + // bulk-insertion-into-empty + auto const flat_set_insert{ time_insertion( flat_set, sorted_numbers ) }; + auto const bpt_insert { time_insertion( bpt , sorted_numbers ) }; +#if HAVE_ABSL + auto const abpt_insert { time_insertion( abpt , sorted_numbers ) }; +#endif + + // random lookup + auto const flat_set_find{ time_lookup( flat_set, numbers ) }; + auto const bpt_find{ time_lookup( bpt , numbers ) }; +#if HAVE_ABSL + auto const abpt_find{ time_lookup( abpt , numbers ) }; +#endif + + std::println( "insert / lookup:" ); + std::println( "\t boost::container::flat_set:\t{} / {}", flat_set_insert, flat_set_find ); + std::println( "\t psi::vm::bpt:\t{} / {}", bpt_insert, bpt_find ); +#if HAVE_ABSL + std::println( "\t absl::bpt:\t{} / {}", abpt_insert, abpt_find ); +#endif + +#if 0 // CI servers are unreliable for comparative perf tests + EXPECT_LE( bpt_find, flat_set_find ); +#endif +} // bp_tree.benchamrk +#endif // release build + +static auto const test_file{ "test.bpt" }; + +TEST( bp_tree, playground ) +{ + // TODO different types, insertion from non contiguous containers + +#ifdef NDEBUG + auto const test_size{ 6853735 }; +#else + auto const test_size{ 258735 }; +#endif + std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; + auto const seed{ std::random_device{}() }; + std::println( "Seed {}", seed ); + std::mt19937 rng{ seed }; + auto numbers{ std::ranges::to( sorted_numbers ) }; + std::span const nums{ numbers }; + // leave the largest quarter of values at the end to trigger/exercise the + // bulk_append branch in insert() + std::ranges::shuffle( nums.subspan( 0, 3 * nums.size() / 4 ), rng ); + std::ranges::shuffle( nums.subspan( 3 * nums.size() / 4 ), rng ); + { + bp_tree bpt; + bpt.map_memory( nums.size() ); + { + auto const third{ nums.size() / 3 }; + auto const first_third{ nums.subspan( 0 * third, third ) }; + auto const second_third{ nums.subspan( 1 * third, third ) }; + auto const third_third{ nums.subspan( 2 * third ) }; + EXPECT_EQ( bpt.insert( first_third ) , first_third.size() ); // bulk into empty + for ( auto const & n : second_third ) { EXPECT_EQ( bpt.insert( n ).second, true ); } // single value + EXPECT_EQ( bpt.insert( third_third ) , third_third.size() ); // bulk into non-empty + + EXPECT_EQ( bpt.insert( second_third ), 0 ); + } + + static_assert( std::forward_iterator::const_iterator> ); + + EXPECT_EQ ( std::distance( bpt. begin(), bpt. end() ), bpt.size() ); + EXPECT_EQ ( std::distance( bpt.ra_begin(), bpt.ra_end() ), bpt.size() ); + EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ), bpt.comp() ) ); + EXPECT_TRUE( std::ranges::equal( sorted_numbers, bpt ) ); + EXPECT_TRUE( std::ranges::equal( sorted_numbers, bpt.random_access() ) ); + EXPECT_NE( bpt.find( +42 ), bpt.end() ); + EXPECT_EQ( bpt.find( -42 ), bpt.end() ); + EXPECT_TRUE ( bpt.erase( 42 ) ); + EXPECT_FALSE( bpt.erase( 42 ) ); + EXPECT_EQ ( bpt.find( +42 ), bpt.end() ); + EXPECT_TRUE ( bpt.insert( 42 ).second ); + EXPECT_FALSE( bpt.insert( 42 ).second ); + EXPECT_EQ ( *bpt.insert( 42 ).first, 42 ); + EXPECT_TRUE( std::ranges::equal( bpt.random_access() , sorted_numbers ) ); + EXPECT_TRUE( std::ranges::equal( std::views::reverse( bpt.random_access() ), std::views::reverse( sorted_numbers ) ) ); + + EXPECT_TRUE( bpt.erase( 42 ) ); + auto const hint42{ bpt.lower_bound( 42 ) }; + EXPECT_EQ( *hint42, 42 + 1 ); + EXPECT_EQ( *bpt.insert( hint42, 42 ), 42 ); + + EXPECT_EQ( *bpt.erase( bpt.find( 42 ) ), 43 ); + EXPECT_TRUE( bpt.insert( 42 ).second ); + + { + auto const ra{ bpt.random_access() }; + for ( auto n : std::views::iota( 0, test_size / 55 ) ) // slow operation (not really amortized constant time): use a smaller subset of the input + EXPECT_EQ( ra[ n ], n ); + } + + // merge test: to exercise the bulk_append path leave/add extra entries + // at the end of input data, bigger than all existing values, that are + // separately shuffled (so that they remain at the end and thus trigger + // the bulk_append branch in merge()). + auto const extra_entries_for_tree_merge{ test_size / 5 }; + std::ranges::iota_view const merge_appendix{ test_size, test_size + extra_entries_for_tree_merge }; + { + auto const evenize{ std::views::transform( []( int const v ) { return v * 2; } ) }; + + auto const even_sorted_numbers{ std::ranges::iota_view{ 0, test_size / 2 } | evenize }; + auto shuffled_even_numbers{ std::ranges::to( even_sorted_numbers ) }; + std::shuffle( shuffled_even_numbers.begin(), shuffled_even_numbers.end(), rng ); + for ( auto const & n : shuffled_even_numbers ) { + EXPECT_TRUE( bpt.erase( n ) ); + } + + bp_tree bpt_even; + bpt_even.map_memory(); + shuffled_even_numbers.append_range( merge_appendix ); + bpt_even.insert( shuffled_even_numbers ); + + EXPECT_EQ( bpt.merge( std::move( bpt_even ) ), bpt_even.size() ); + } + + EXPECT_TRUE( std::ranges::equal( std::ranges::iota_view{ 0, test_size + extra_entries_for_tree_merge }, bpt ) ); + + std::shuffle( numbers.begin(), numbers.end(), rng ); + for ( auto const & n : numbers ) + EXPECT_TRUE( bpt.erase( n ) ); + for ( auto const & n : merge_appendix ) { + // + auto const next_it{ bpt.erase( bpt.find( n ) ) }; + EXPECT_TRUE( ( next_it == bpt.end() ) || ( *next_it == n + 1 ) ); + } + + EXPECT_TRUE( bpt.empty() ); + } + + { + bp_tree bpt; + bpt.map_file( test_file, flags::named_object_construction_policy::create_new_or_truncate_existing ); + + for ( auto const & n : numbers ) + EXPECT_TRUE( bpt.insert( n ).second ); + + static_assert( std::forward_iterator::const_iterator> ); + + EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ), bpt.comp() ) ); + EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); + EXPECT_NE ( bpt.find( +42 ), bpt.end() ); + EXPECT_EQ ( bpt.find( -42 ), bpt.end() ); + EXPECT_TRUE( bpt.erase( 42 ) ); + EXPECT_EQ ( bpt.find( +42 ), bpt.end() ); + } + { + bp_tree bpt; + bpt.map_file( test_file, flags::named_object_construction_policy::open_existing ); + + EXPECT_EQ ( bpt.size(), sorted_numbers.size() - 1 ); + EXPECT_TRUE( bpt.insert( +42 ).second ); + + EXPECT_TRUE( std::ranges::is_sorted( bpt, bpt.comp() ) ); + EXPECT_TRUE( std::ranges::equal( std::as_const( bpt ), sorted_numbers ) ); + EXPECT_NE( bpt.find( +42 ), bpt.end() ); + EXPECT_EQ( bpt.find( -42 ), bpt.end() ); + + bpt.clear(); + bpt.print(); + } +} + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/test/vector.cpp b/test/vector.cpp index 49a4906..3be0649 100644 --- a/test/vector.cpp +++ b/test/vector.cpp @@ -2,21 +2,35 @@ #include -#include #include -#include //------------------------------------------------------------------------------ namespace psi::vm { //------------------------------------------------------------------------------ -TEST( vector, playground ) +TEST( vector, anon_memory_backed ) { - std::filesystem::path const test_vec{ "test_vec" }; - std::filesystem::remove( test_vec ); + psi::vm::vector vec; + vec.map_memory(); + EXPECT_EQ( vec.size(), 0 ); + vec.append_range({ 3.14, 0.14, 0.04 }); + EXPECT_EQ( vec.size(), 3 ); + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); + vec.grow_by( 12345678, default_init ); + // test growth (with 'probable' relocation) does not destroy contents + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); +} + +TEST( vector, file_backed ) +{ + auto const test_vec{ "test.vec" }; { - psi::vm::vector< double, std::uint16_t > vec; - vec.map_file( test_vec.c_str(), flags::named_object_construction_policy::create_new ); + psi::vm::vector vec; + vec.map_file( test_vec, flags::named_object_construction_policy::create_new_or_truncate_existing ); EXPECT_EQ( vec.size(), 0 ); vec.append_range({ 3.14, 0.14, 0.04 }); EXPECT_EQ( vec.size(), 3 ); @@ -25,8 +39,8 @@ TEST( vector, playground ) EXPECT_EQ( vec[ 2 ], 0.04 ); } { - psi::vm::vector< double, std::uint16_t > vec; - vec.map_file( test_vec.c_str(), flags::named_object_construction_policy::open_existing ); + psi::vm::vector vec; + vec.map_file( test_vec, flags::named_object_construction_policy::open_existing ); EXPECT_EQ( vec.size(), 3 ); EXPECT_EQ( vec[ 0 ], 3.14 ); EXPECT_EQ( vec[ 1 ], 0.14 );