From db8666836bfab7c2408651bf8a0395aee34d0b4f Mon Sep 17 00:00:00 2001 From: psiha Date: Fri, 27 Sep 2024 14:09:34 +0200 Subject: [PATCH 01/49] Windows: - implemented support (emulation) for unmapping views expanded by concatenation - fixed NTSTATUS type definition - added a minor sanity check. --- include/psi/vm/detail/nt.hpp | 2 +- src/allocation/allocation.impl.hpp | 19 +++++------- src/allocation/allocation.posix.cpp | 1 + src/allocation/allocation.win32.cpp | 37 ++++++++++++------------ src/allocation/remap.cpp | 1 + src/mappable_objects/file/file.win32.cpp | 1 + src/mapped_view/mapped_view.cpp | 2 +- src/mapped_view/mapped_view.win32.cpp | 36 +++++++++++++---------- src/mapping/mapping.win32.cpp | 2 +- 9 files changed, 52 insertions(+), 49 deletions(-) diff --git a/include/psi/vm/detail/nt.hpp b/include/psi/vm/detail/nt.hpp index b4f57df..59d0645 100644 --- a/include/psi/vm/detail/nt.hpp +++ b/include/psi/vm/detail/nt.hpp @@ -31,7 +31,7 @@ namespace psi::vm::nt //------------------------------------------------------------------------------ #ifndef STATUS_SUCCESS -using NTSTATUS = DWORD; +using NTSTATUS = LONG; NTSTATUS constexpr STATUS_SUCCESS{ 0 }; #endif // STATUS_SUCCESS #ifndef STATUS_CONFLICTING_ADDRESSES diff --git a/src/allocation/allocation.impl.hpp b/src/allocation/allocation.impl.hpp index 9bffa3c..50c1ab0 100644 --- a/src/allocation/allocation.impl.hpp +++ b/src/allocation/allocation.impl.hpp @@ -13,26 +13,21 @@ //------------------------------------------------------------------------------ #pragma once -#include +#include //------------------------------------------------------------------------------ -namespace psi -{ -//------------------------------------------------------------------------------ -namespace vm +namespace psi::vm { //------------------------------------------------------------------------------ namespace // internal header { - constexpr void * add( void * const ptr, auto const diff ) noexcept { return static_cast( ptr ) + diff; } - constexpr void const * add( void const * const ptr, auto const diff ) noexcept { return static_cast( ptr ) + diff; } - constexpr auto add( auto const x , auto const diff ) noexcept { return x + diff; } + constexpr void add( void * & ptr, auto const diff ) noexcept { reinterpret_cast( ptr ) += diff; } + constexpr void add( void const * & ptr, auto const diff ) noexcept { reinterpret_cast( ptr ) += diff; } + constexpr void add( auto & x , auto const diff ) noexcept { x += diff; } - constexpr auto sub( auto const x , auto const diff ) noexcept { return x - diff; } + constexpr void sub( auto & x , auto const diff ) noexcept { x -= diff; } } // anonymous namespace //------------------------------------------------------------------------------ -} // namespace vm -//------------------------------------------------------------------------------ -} // namespace psi +} // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/src/allocation/allocation.posix.cpp b/src/allocation/allocation.posix.cpp index 262c66a..ff8736c 100644 --- a/src/allocation/allocation.posix.cpp +++ b/src/allocation/allocation.posix.cpp @@ -12,6 +12,7 @@ //////////////////////////////////////////////////////////////////////////////// //------------------------------------------------------------------------------ #include "allocation.impl.hpp" +#include #include #include diff --git a/src/allocation/allocation.win32.cpp b/src/allocation/allocation.win32.cpp index e80e235..7977c4f 100644 --- a/src/allocation/allocation.win32.cpp +++ b/src/allocation/allocation.win32.cpp @@ -12,6 +12,7 @@ //////////////////////////////////////////////////////////////////////////////// //------------------------------------------------------------------------------ #include "allocation.impl.hpp" +#include #include #include @@ -24,8 +25,6 @@ namespace psi::vm { //------------------------------------------------------------------------------ -auto const this_process{ reinterpret_cast( static_cast( -1 ) ) }; - enum class deallocation_type : std::uint32_t { free = MEM_RELEASE, @@ -45,7 +44,7 @@ auto alloc( void * & desired_location, std::size_t & size, allocation_type const { NtAllocateVirtualMemory ( - this_process, + nt::current_process, reinterpret_cast< PVOID * >( &desired_location ), 0, // address zero bits &sz, @@ -65,7 +64,7 @@ auto alloc( void * & desired_location, std::size_t & size, allocation_type const } else { - BOOST_ASSUME( ( nt_status == STATUS_NO_MEMORY ) || ( nt_status == STATUS_INVALID_PARAMETER && !size ) ); + BOOST_ASSUME( ( nt_status == NTSTATUS( STATUS_NO_MEMORY ) ) || ( nt_status == NTSTATUS( STATUS_INVALID_PARAMETER ) && !size ) ); desired_location = nullptr; } return nt_status; @@ -86,7 +85,7 @@ void dealloc( void * & address, std::size_t & size, deallocation_type const type #if NATIVE_NT using namespace nt; SIZE_T sz( size ); - auto const result{ NtFreeVirtualMemory( this_process, &address, &sz, std::to_underlying( type ) ) }; + auto const result{ NtFreeVirtualMemory( nt::current_process, &address, &sz, std::to_underlying( type ) ) }; size = static_cast< std::size_t >( sz ); BOOST_ASSUME( result == STATUS_SUCCESS || size == 0 ); #else @@ -94,11 +93,11 @@ void dealloc( void * & address, std::size_t & size, deallocation_type const type #endif } -auto mem_info( void * const pAddress ) noexcept +MEMORY_BASIC_INFORMATION mem_info( void * const address ) noexcept { MEMORY_BASIC_INFORMATION info; - BOOST_VERIFY( ::VirtualQueryEx( this_process, pAddress, &info, sizeof( info ) ) == sizeof( info ) ); - BOOST_ASSUME( info.BaseAddress == pAddress ); + BOOST_VERIFY( ::VirtualQueryEx( nt::current_process, address, &info, sizeof( info ) ) == sizeof( info ) ); + BOOST_ASSUME( info.BaseAddress == address ); return info; } @@ -123,17 +122,17 @@ bool commit( void * const desired_location, std::size_t const size ) noexcept BOOST_ASSUME( info.State == MEM_RESERVE ); BOOST_ASSUME( info.Protect == 0 ); BOOST_ASSUME( info.Type == MEM_PRIVATE ); - auto regionSize{ std::min( static_cast< std::size_t >( info.RegionSize ), size - final_size ) }; - auto const partial_result{ alloc( final_address, regionSize, allocation_type::commit ) }; + auto region_size{ std::min( static_cast< std::size_t >( info.RegionSize ), size - final_size ) }; + auto const partial_result{ alloc( final_address, region_size, allocation_type::commit ) }; if ( partial_result != STATUS_SUCCESS ) { - BOOST_ASSERT( partial_result == STATUS_NO_MEMORY ); + BOOST_ASSERT( partial_result == NTSTATUS( STATUS_NO_MEMORY ) ); return false; } - BOOST_ASSUME( regionSize <= info.RegionSize ); + BOOST_ASSUME( region_size <= info.RegionSize ); - add( final_address, regionSize ); - add( final_size , regionSize ); + add( final_address, region_size ); + add( final_size , region_size ); } final_address = desired_location; result = STATUS_SUCCESS; @@ -158,11 +157,11 @@ void free( void * address, std::size_t size ) noexcept // emulate support for adjacent/concatenated regions (as supported by mmap) while ( size ) { - std::size_t releasedSize{ 0 }; - dealloc( address, releasedSize, deallocation_type::free ); - BOOST_ASSERT( releasedSize <= size ); - add( address, releasedSize ); - sub( size , releasedSize ); + std::size_t released_size{ 0 }; + dealloc( address, released_size, deallocation_type::free ); + BOOST_ASSUME( released_size <= size ); + add( address, released_size ); + sub( size , released_size ); } } diff --git a/src/allocation/remap.cpp b/src/allocation/remap.cpp index 21cf0b0..bdac147 100644 --- a/src/allocation/remap.cpp +++ b/src/allocation/remap.cpp @@ -13,6 +13,7 @@ //------------------------------------------------------------------------------ #include "allocation.impl.hpp" #include +#include #include #include diff --git a/src/mappable_objects/file/file.win32.cpp b/src/mappable_objects/file/file.win32.cpp index e53feb4..a59753e 100644 --- a/src/mappable_objects/file/file.win32.cpp +++ b/src/mappable_objects/file/file.win32.cpp @@ -117,6 +117,7 @@ namespace detail { HANDLE handle{ handle_traits::invalid_value }; LARGE_INTEGER maximum_size{ .QuadPart = static_cast( size ) }; + BOOST_ASSERT_MSG( std::uint64_t( maximum_size.QuadPart ) == size, "Unsupported section size" ); auto const nt_result { nt::NtCreateSection // TODO use it for named sections also diff --git a/src/mapped_view/mapped_view.cpp b/src/mapped_view/mapped_view.cpp index e25fb76..0341821 100644 --- a/src/mapped_view/mapped_view.cpp +++ b/src/mapped_view/mapped_view.cpp @@ -140,7 +140,7 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o #else ULARGE_INTEGER const win32_offset{ .QuadPart = target_offset }; auto const new_address - { // TODO this (appending) requires complexity to be added on the unmap side (see mapper::unmap) + { ::MapViewOfFileEx ( original_mapping.get(), diff --git a/src/mapped_view/mapped_view.win32.cpp b/src/mapped_view/mapped_view.win32.cpp index cabae35..9793a57 100644 --- a/src/mapped_view/mapped_view.win32.cpp +++ b/src/mapped_view/mapped_view.win32.cpp @@ -14,6 +14,7 @@ /// //////////////////////////////////////////////////////////////////////////////// //------------------------------------------------------------------------------ +#include "../allocation/allocation.impl.hpp" #include #include #include @@ -98,29 +99,34 @@ map #endif } +// defined in allocation.win32.cpp +MEMORY_BASIC_INFORMATION mem_info( void * const ) noexcept; + namespace { - [[ maybe_unused ]] - auto mem_info( void * const address ) noexcept + BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_RESTRICTED_FUNCTION_L1 ) BOOST_NOINLINE + void unmap_concatenated( void * address, std::size_t size ) noexcept { - MEMORY_BASIC_INFORMATION info; - BOOST_VERIFY( ::VirtualQueryEx( nt::current_process, address, &info, sizeof( info ) ) == sizeof( info ) ); - return info; + // emulate support for adjacent/concatenated regions (as supported by mmap) + // (duplicated logic from free() in allocaiton.win32.cpp) + for ( ;; ) + { + auto const region_size{ mem_info( address ).RegionSize }; + BOOST_ASSUME( region_size <= align_up( size, reserve_granularity ) ); + BOOST_VERIFY( ::UnmapViewOfFile( address ) ); + if ( region_size >= size ) + break; + add( address, region_size ); + sub( size , region_size ); + } } } // anonymous namespace -BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_RESTRICTED_FUNCTION_L1 ) +BOOST_ATTRIBUTES( BOOST_EXCEPTIONLESS, BOOST_RESTRICTED_FUNCTION_L1 ) void unmap( mapped_span const view ) noexcept { - BOOST_ASSERT_MSG - ( // greater-or-equal because of the inability of Windows to do a proper unmap_partial - // (so we do discard() calls and shrinking of the view-span only, while the actual - // region remains in its original size) - mem_info( view.data() ).RegionSize >= align_up( view.size(), commit_granularity ) || view.empty(), - "TODO: implement a loop to handle concatenated ranges" - // see mapped_view::expand, Windows does not allow partial unmaps or merging of mapped views/ranges - ); - BOOST_VERIFY( ::UnmapViewOfFile( view.data() ) || view.empty() ); + if ( !view.empty() ) + unmap_concatenated( view.data(), view.size() ); } void unmap_partial( mapped_span const range ) noexcept diff --git a/src/mapping/mapping.win32.cpp b/src/mapping/mapping.win32.cpp index 34ba362..d4efb54 100644 --- a/src/mapping/mapping.win32.cpp +++ b/src/mapping/mapping.win32.cpp @@ -40,7 +40,7 @@ std::uint64_t get_size( mapping::const_handle const mapping_handle ) noexcept namespace detail::create_mapping_impl { HANDLE map_file( file_handle::reference file, flags::flags_t flags, std::uint64_t size ) noexcept; } -err::fallible_result set_size( mapping & __restrict the_mapping, std::uint64_t const new_size ) noexcept +err::fallible_result set_size( mapping & the_mapping, std::uint64_t const new_size ) noexcept { using namespace nt; LARGE_INTEGER ntsz{ .QuadPart = static_cast( new_size ) }; From f43f166b4037931a33357f189f61f56b251723b8 Mon Sep 17 00:00:00 2001 From: psiha Date: Fri, 27 Sep 2024 14:42:01 +0200 Subject: [PATCH 02/49] First working B+Tree prototype. Build: - added PCH usage - updated dependencies. --- CMakeLists.txt | 6 +- include/psi/vm/containers/b+tree.hpp | 1019 ++++++++++++++++++++ include/psi/vm/containers/b+tree_print.hpp | 82 ++ include/psi/vm/vector.hpp | 14 +- src/containers/b+tree.cpp | 171 ++++ src/pch.hpp | 34 + test/CMakeLists.txt | 4 +- test/b+tree.cpp | 59 ++ 8 files changed, 1381 insertions(+), 8 deletions(-) create mode 100644 include/psi/vm/containers/b+tree.hpp create mode 100644 include/psi/vm/containers/b+tree_print.hpp create mode 100644 src/containers/b+tree.cpp create mode 100644 src/pch.hpp create mode 100644 test/b+tree.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 274fb4e..6b8bc9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ include( ${CMAKE_CURRENT_BINARY_DIR}/cmake/get_cpm.cmake ) # Add packages -set( boost_ver boost-1.85.0 ) +set( boost_ver boost-1.86.0 ) CPMAddPackage( "gh:boostorg/static_assert#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/throw_exception#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/config#${boost_ver}" ) # Boost::core dependency @@ -29,6 +29,7 @@ CPMAddPackage( "gh:boostorg/predef#${boost_ver}" ) # Boost::winapi depe CPMAddPackage( "gh:boostorg/assert#${boost_ver}" ) CPMAddPackage( "gh:boostorg/core#${boost_ver}" ) CPMAddPackage( "gh:boostorg/preprocessor#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/stl_interfaces#${boost_ver}" ) CPMAddPackage( "gh:boostorg/winapi#${boost_ver}" ) CPMAddPackage( "gh:boostorg/utility#${boost_ver}" ) @@ -72,6 +73,7 @@ target_link_libraries( psi_vm PUBLIC Boost::core Boost::assert Boost::preprocessor + Boost::stl_interfaces Boost::winapi Boost::utility ) @@ -83,6 +85,8 @@ target_include_directories( psi_vm PUBLIC "${std_fix_SOURCE_DIR}/include" # vm::vector uses it (in a header) ) +target_precompile_headers( psi_vm PUBLIC src/pch.hpp ) + ################### ## Testing diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp new file mode 100644 index 0000000..a8ac721 --- /dev/null +++ b/include/psi/vm/containers/b+tree.hpp @@ -0,0 +1,1019 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +namespace detail +{ + template + [[ gnu::const ]] auto header_data( std::span const hdr_storage ) noexcept + { + auto const data { align_up( hdr_storage.data() ) }; + auto const remaining_space{ hdr_storage.size() - unsigned( data - hdr_storage.data() ) }; + BOOST_ASSERT( remaining_space >= sizeof( Header ) ); + return std::pair{ reinterpret_cast
( data ), std::span{ data + sizeof( Header ), remaining_space - sizeof( Header ) } }; + } +} // namespace detail + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base +//////////////////////////////////////////////////////////////////////////////// + +class bptree_base +{ +public: + using size_type = std::size_t; + using difference_type = std::make_signed_t; + using storage_result = err::fallible_result; + + [[ gnu::pure ]] auto empty() const noexcept { return size() == 0; } + + void clear() noexcept; + + bptree_base( header_info hdr_info = {} ) noexcept; + + std::span user_header_data() noexcept; + + bool has_attached_storage() const noexcept { return nodes_.has_attached_storage(); } + + storage_result map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { return init_header( nodes_.map_file ( file, policy ) ); } + storage_result map_memory( size_type const initial_capacity_in_nodes = 0 ) noexcept { return init_header( nodes_.map_memory( initial_capacity_in_nodes ) ); } + +protected: + static constexpr auto node_size{ page_size }; + + using depth_t = std::uint8_t; + + struct [[ nodiscard, clang::trivial_abi ]] node_slot // instead of node pointers we store offsets - slots in the node pool + { + using value_type = std::uint32_t; + static node_slot const null; + value_type index{ static_cast( -1 ) }; // in-pool index/offset + [[ gnu::pure ]] value_type operator*() const noexcept { return index; } + [[ gnu::pure ]] bool operator==( node_slot const other ) const noexcept { return this->index == other.index; } + [[ gnu::pure ]] explicit operator bool() const noexcept { return index != null.index; } + }; // struct node_slot + + struct [[ nodiscard ]] node_header + { + using size_type = std::uint16_t; + + node_slot parent {}; + size_type num_vals{}; + /* TODO + size_type start; // make keys and children arrays function as devectors: allow empty space at the beginning to avoid moves for small borrowings + size_type in_parent_pos; // position of the node in its parent (to avoid finds in upscans). + */ + + [[ gnu::pure ]] bool is_root() const noexcept { return !parent; } + + // merely to prevent slicing (in return-node-by-ref cases) + node_header( node_header const & ) = delete; + constexpr node_header( node_header && ) noexcept = default; + constexpr node_header( ) noexcept = default; + constexpr node_header & operator=( node_header && ) noexcept = default; + }; // struct node_header + using node_size_type = node_header::size_type; + + struct linked_node_header : node_header { node_slot next{}; /*singly linked free list*/ }; + + struct alignas( node_size ) node_placeholder : node_header {}; + struct alignas( node_size ) free_node : linked_node_header {}; + + // SCARY iterator parts + class base_iterator; + class base_random_access_iterator; + + struct header + { + node_slot root_; + node_slot free_list_; + node_slot leaves_; + size_t size_; + depth_t depth_; + }; // struct header + + using node_pool = vm::vector; + +protected: + void swap( bptree_base & other ) noexcept; + + [[ gnu::pure ]] size_type size() const noexcept { return hdr().size_; } + + [[ gnu::cold ]] linked_node_header & create_root(); + + [[ gnu::pure ]] static bool underflowed( auto const & node ) noexcept { return node.num_vals < node.min_values; } + [[ gnu::pure ]] static bool can_borrow ( auto const & node ) noexcept { return node.num_vals > node.min_values; } + + [[ gnu::pure ]] depth_t leaf_level( ) const noexcept; + [[ gnu::pure ]] bool is_leaf_level( depth_t const level ) const noexcept; + + void free( node_header & ) noexcept; + + [[ gnu::pure ]] header & hdr() noexcept; + [[ gnu::pure ]] header const & hdr() const noexcept { return const_cast( *this ).hdr(); } + + node_slot first_leaf() const noexcept { return hdr().leaves_; } + + static void verify( auto const & node ) noexcept + { + BOOST_ASSERT( std::ranges::is_sorted( keys( node ) ) ); + BOOST_ASSUME( node.num_vals <= node.max_values ); + // also used for underflowing nodes and (most problematically) for root nodes 'interpreted' as inner nodes...TODO... + //BOOST_ASSUME( node.num_vals >= node.min_values ); + } + + static constexpr auto keys ( auto & node ) noexcept { return std::span{ node.keys , node.num_vals }; } + static constexpr auto keys ( auto const & node ) noexcept { return std::span{ node.keys , node.num_vals }; } + static constexpr auto children( auto & node ) noexcept { if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } + static constexpr auto children( auto const & node ) noexcept { if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } + + [[ gnu::pure ]] static constexpr node_size_type num_vals ( auto const & node ) noexcept { return node.num_vals; } + [[ gnu::pure ]] static constexpr node_size_type num_chldrn( auto const & node ) noexcept { if constexpr ( requires{ node.children; } ) { BOOST_ASSUME( node.num_vals ); return node.num_vals + 1U; } else return 0; } + + template + static NodeType & as( SourceNode & slot ) noexcept + { + static_assert( sizeof( NodeType ) == sizeof( slot ) ); + return static_cast( static_cast( slot ) ); + } + template + static NodeType const & as( SourceNode const & slot ) noexcept { return as( const_cast( slot ) ); } + + node_placeholder & node( node_slot const offset ) noexcept { return nodes_[ *offset ]; } + node_placeholder const & node( node_slot const offset ) const noexcept { return nodes_[ *offset ]; } + + auto & root() noexcept { return node( hdr().root_ ); } + auto const & root() const noexcept { return const_cast( *this ).root(); } + + template N & node( node_slot const offset ) noexcept { return as( node( offset ) ); } + template N const & node( node_slot const offset ) const noexcept { return as( node( offset ) ); } + + [[ gnu::pure ]] node_slot slot_of( node_header const & ) const noexcept; + + static bool full( auto const & node ) noexcept { + BOOST_ASSUME( node.num_vals <= node.max_values ); + return node.num_vals == node.max_values; + } + + template + static void umove // uninitialized move + ( + N const & source, node_size_type const srcBegin, node_size_type const srcEnd, + N & target, node_size_type const tgtBegin + ) noexcept { + BOOST_ASSUME( srcBegin <= srcEnd ); + std::uninitialized_move( &(source.*array)[ srcBegin ], &(source.*array)[ srcEnd ], &(target.*array)[ tgtBegin ] ); + } + + [[ nodiscard ]] node_placeholder & new_node(); + + template + [[ nodiscard ]] N & new_node() { return as( new_node() ); } + +private: + auto header_data() noexcept { return detail::header_data
( nodes_.user_header_data() ); } + + storage_result init_header( storage_result ) noexcept; + +protected: + node_pool nodes_; +}; // class bptree_base + +inline constexpr bptree_base::node_slot const bptree_base::node_slot::null{ static_cast( -1 ) }; + + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base::base_iterator +//////////////////////////////////////////////////////////////////////////////// + +class bptree_base::base_iterator +{ +protected: +#ifndef NDEBUG // for bounds checking + std::span +#else + node_placeholder * __restrict +#endif + nodes_ {}; + node_slot node_slot_ {}; + node_size_type value_offset_{}; + + template friend class bp_tree; + + base_iterator( node_pool &, node_slot, node_size_type value_offset ) noexcept; + + [[ gnu::pure ]] linked_node_header & node() const noexcept; + +public: + constexpr base_iterator() noexcept = default; + + base_iterator & operator++() noexcept; + + bool operator==( base_iterator const & ) const noexcept; +}; // class base_iterator + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base::base_random_access_iterator +//////////////////////////////////////////////////////////////////////////////// + +class bptree_base::base_random_access_iterator : public base_iterator +{ +protected: + size_type index_; + node_slot leaves_start_; // merely for the rewind for negative offsets in operator+= + + template friend class bp_tree; + + base_random_access_iterator( bptree_base & parent, node_slot const start_leaf, size_type const start_index ) noexcept + : base_iterator{ parent.nodes_, start_leaf, 0 }, index_{ start_index }, leaves_start_{ parent.first_leaf() } {} + +public: + constexpr base_random_access_iterator() noexcept = default; + + difference_type operator-( base_random_access_iterator const & other ) const noexcept { return static_cast( this->index_ - other.index_ ); } + + base_random_access_iterator & operator+=( difference_type n ) noexcept; + + base_random_access_iterator & operator++( ) noexcept { base_iterator::operator++(); ++index_; return *this; } + base_random_access_iterator operator++(int) noexcept { auto current{ *this }; operator++(); return current; } + + // should implicitly handle end iterator comparison also (this requires the start_index constructor argument for the construction of end iterators) + friend constexpr bool operator==( base_random_access_iterator const & left, base_random_access_iterator const & right ) noexcept { return left.index_ == right.index_; } +}; // class base_random_access_iterator + + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base_wkey +//////////////////////////////////////////////////////////////////////////////// + +template +class bptree_base_wkey : public bptree_base +{ +public: + using key_type = Key; + using value_type = key_type; // TODO map support + + using key_const_arg = std::conditional_t, Key, Key const &>; + +protected: // node types + struct alignas( node_size ) parent_node : node_header + { + static auto constexpr storage_space{ node_size - psi::vm::align_up( sizeof( node_header ), alignof( Key ) ) }; + + // storage_space = ( order - 1 ) * sizeof( key ) + order * sizeof( child_ptr ) + // storage_space = order * szK - szK + order * szC + // storage_space + szK = order * ( szK + szC ) + // order = ( storage_space + szK ) / ( szK + szC ) + static constexpr node_size_type order // "m" + { + ( storage_space + sizeof( Key ) ) + / + ( sizeof( Key ) + sizeof( node_slot ) ) + }; + + using value_type = Key; + + static auto constexpr max_children{ order }; + static auto constexpr max_values { max_children - 1 }; + + node_slot children[ max_children ]; + Key keys [ max_values ]; + }; // struct inner_node + + struct inner_node : parent_node + { + static auto constexpr min_children{ ( parent_node::max_children + 1 ) / 2 }; // ceil( m / 2 ) + static auto constexpr min_values { min_children - 1 }; + + static_assert( min_children >= 3 ); + }; // struct inner_node + + struct root_node : parent_node + { + static auto constexpr min_children{ 2 }; + static auto constexpr min_values { min_children - 1 }; + }; // struct root_node + + struct alignas( node_size ) leaf_node : linked_node_header + { + // TODO support for maps (i.e. keys+values) + using value_type = Key; + + static auto constexpr storage_space{ node_size - psi::vm::align_up( sizeof( linked_node_header ), alignof( Key ) ) }; + static auto constexpr max_values { storage_space / sizeof( Key ) }; + static auto constexpr min_values { ( max_values + 1 ) / 2 }; + + Key keys[ max_values ]; + }; // struct leaf_node + + static_assert( sizeof( inner_node ) == node_size ); + static_assert( sizeof( leaf_node ) == node_size ); + +protected: // iterators + template + using iter_impl = boost::stl_interfaces::iterator_interface + < +# if !BOOST_STL_INTERFACES_USE_DEDUCED_THIS + Impl, +# endif + Tag, + Key + >; + + class fwd_iterator; + class ra_iterator; + +protected: // helpers for split_to_insert + static value_type insert_into_new_node + ( + parent_node & node, parent_node & new_node, + key_const_arg value, + node_size_type const insert_pos, node_size_type const new_insert_pos, + node_slot const key_right_child + ) noexcept { + BOOST_ASSUME( bool( key_right_child ) ); + + using N = parent_node; + + auto const max{ N::max_values }; // or 'order' + auto const mid{ max / 2 }; + + value_type key_to_propagate; + if ( new_insert_pos == 0 ) { + umove<&N::keys >( node, mid , node.num_vals , new_node, 0 ); + umove<&N::children>( node, mid + 1, node.num_vals + 1, new_node, 1 ); + children( new_node )[ new_insert_pos ] = key_right_child; + key_to_propagate = std::move( value ); + } else { + key_to_propagate = std::move( node.keys[ mid ] ); + + umove<&N::keys >( node, mid + 1, insert_pos , new_node, 0 ); + umove<&N::children>( node, mid + 1, insert_pos + 1, new_node, 0 ); + + umove<&N::keys >( node, insert_pos , node.num_vals , new_node, new_insert_pos ); + umove<&N::children>( node, insert_pos + 1, node.num_vals + 1, new_node, new_insert_pos + 1 ); + + keys ( new_node )[ new_insert_pos - 1 ] = value; + children( new_node )[ new_insert_pos ] = key_right_child; + } + + node .num_vals = mid ; + new_node.num_vals = max - mid; + + return key_to_propagate; + } + + static value_type const & insert_into_new_node + ( + leaf_node & node, leaf_node & new_node, + key_const_arg value, + node_size_type const insert_pos, node_size_type const new_insert_pos, + node_slot const key_right_child + ) noexcept { + BOOST_ASSUME( !key_right_child ); + + using N = leaf_node; + + auto const max{ N::max_values }; // or 'order' + auto const mid{ max / 2 }; + + umove<&N::keys>( node, mid , insert_pos , new_node, 0 ); + umove<&N::keys>( node, insert_pos, node.num_vals, new_node, new_insert_pos + 1 ); + + node .num_vals = mid ; + new_node.num_vals = max - mid + 1; + + keys( new_node )[ new_insert_pos ] = value; + auto const & key_to_propagate{ new_node.keys[ 0 ] }; + return key_to_propagate; + } + + static value_type insert_into_existing_node( parent_node & node, parent_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + { + BOOST_ASSUME( bool( key_right_child ) ); + + using N = parent_node; + + auto const max{ N::max_values }; // or 'order' + auto const mid{ max / 2 }; + + value_type key_to_propagate{ std::move( node.keys[ mid - 1 ] ) }; + + umove<&N::keys >( node, mid, node.num_vals , new_node, 0 ); + umove<&N::children>( node, mid, node.num_vals + 1, new_node, 0 ); + + umove<&N::keys >( node, insert_pos , mid - 1, node, insert_pos + 1 ); + umove<&N::children>( node, insert_pos + 1, mid , node, insert_pos + 2 ); + + keys ( node )[ insert_pos ] = value; + children( node )[ insert_pos + 1 ] = key_right_child; + + node .num_vals = mid; + new_node.num_vals = max - mid; + + return key_to_propagate; + } + + static value_type const & insert_into_existing_node( leaf_node & node, leaf_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + { + BOOST_ASSUME( !key_right_child ); + + using N = leaf_node; + + auto const max{ N::max_values }; // or 'order' + auto const mid{ max / 2 }; + + umove<&N::keys>( node, mid , node.num_vals, new_node, 0 ); + umove<&N::keys>( node, insert_pos, mid , node , insert_pos + 1 ); + + node .num_vals = mid + 1 ; + new_node.num_vals = max - mid; + + keys( node )[ insert_pos ] = value; + auto const & key_to_propagate{ new_node.keys[ 0 ] }; + return key_to_propagate; + } + +protected: // 'other' + root_node & root() noexcept { return as( bptree_base::root() ); } + root_node const & root() const noexcept { return const_cast( *this ).root(); } + + using bptree_base::free; + void free( leaf_node & node ) noexcept { + auto & first_leaf{ hdr().leaves_ }; + if ( first_leaf == slot_of( node ) ) { + first_leaf = node.next; + } + bptree_base::free( static_cast( node ) ); + } + + void merge_nodes + ( + leaf_node & __restrict source, leaf_node & __restrict target, + inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx + ) noexcept + { + BOOST_ASSUME( source.num_vals <= source.max_values ); BOOST_ASSUME( source.num_vals >= source.min_values - 1 ); + BOOST_ASSUME( target.num_vals <= target.max_values ); BOOST_ASSUME( target.num_vals >= target.min_values - 1 ); + + std::ranges::move( keys ( source ), keys ( target ).end() ); + std::ranges::move( children( source ), children( target ).end() ); + target.num_vals += source.num_vals; + source.num_vals = 0; + target.next = source.next; + BOOST_ASSUME( target.num_vals == target.max_values - 1 ); + + std::ranges::shift_left( keys ( parent ).subspan( parent_key_idx ), 1 ); + std::ranges::shift_left( children( parent ).subspan( parent_child_idx ), 1 ); + BOOST_ASSUME( parent.num_vals ); + parent.num_vals--; + + source.next = slot_of( target ); + verify( target ); + free ( source ); + verify( parent ); + } + + void merge_nodes + ( + inner_node & __restrict right, inner_node & __restrict left, + inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx + ) noexcept + { + BOOST_ASSUME( right.num_vals <= right.max_values ); BOOST_ASSUME( right.num_vals >= right.min_values - 1 ); + BOOST_ASSUME( left .num_vals <= left .max_values ); BOOST_ASSUME( left .num_vals >= left .min_values - 1 ); + + for ( auto const ch_slot : children( right ) ) + this->node( ch_slot ).parent = slot_of( left ); + + std::ranges::move( children( right ), children( left ).end() ); + left.num_vals += 1; + auto & separator_key{ parent.keys[ parent_key_idx ] }; + keys( left ).back() = std::move( separator_key ); + std::ranges::move( keys( right ), keys( left ).end() ); + + left .num_vals += right.num_vals; + right.num_vals = 0; + BOOST_ASSUME( left.num_vals == left.max_values ); + verify( left ); + free ( right ); + + std::ranges::shift_left( keys ( parent ).subspan( parent_key_idx ), 1 ); + std::ranges::shift_left( children( parent ).subspan( parent_child_idx ), 1 ); + BOOST_ASSUME( parent.num_vals ); + parent.num_vals--; + } + +public: + // solely a debugging helper + void print() const; +}; // class bptree_base_wkey + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base_wkey::fwd_iterator +//////////////////////////////////////////////////////////////////////////////// + +template +class bptree_base_wkey::fwd_iterator + : + public base_iterator, + public iter_impl +{ +private: + using impl = iter_impl; + + using base_iterator::base_iterator; + +public: + constexpr fwd_iterator() noexcept = default; + + Key & operator*() const noexcept { return static_cast( node() ).keys[ value_offset_ ]; } + + constexpr fwd_iterator & operator++() noexcept { return static_cast( base_iterator::operator++() ); } + using impl::operator++; +}; // class fwd_iterator + +//////////////////////////////////////////////////////////////////////////////// +// \class bptree_base_wkey::ra_iterator +//////////////////////////////////////////////////////////////////////////////// + +template +class bptree_base_wkey::ra_iterator + : + public base_random_access_iterator, + public iter_impl +{ +private: + using base_random_access_iterator::base_random_access_iterator; + +public: + constexpr ra_iterator() noexcept = default; + + Key & operator*() const noexcept { return static_cast( node() ).keys[ value_offset_ ]; } + + ra_iterator & operator+=( difference_type const n ) noexcept { return static_cast( base_random_access_iterator::operator+=( n ) ); } + + ra_iterator & operator++( ) noexcept { return static_cast( base_random_access_iterator::operator++( ) ); } + ra_iterator operator++(int) noexcept { return static_cast( base_random_access_iterator::operator++(0) ); } + + friend constexpr bool operator==( ra_iterator const & left, ra_iterator const & right ) noexcept { return left.index_ == right.index_; } + + operator fwd_iterator() const noexcept { return static_cast( static_cast( *this ) ); } +}; // class ra_iterator + + +//////////////////////////////////////////////////////////////////////////////// +// \class bp_tree +//////////////////////////////////////////////////////////////////////////////// + +template > +class bp_tree + : + public bptree_base_wkey, + private Comparator, + public boost::stl_interfaces::sequence_container_interface, boost::stl_interfaces::element_layout::discontiguous> +{ +private: + using base = bptree_base_wkey; + using stl_impl = boost::stl_interfaces::sequence_container_interface, boost::stl_interfaces::element_layout::discontiguous>; + + using depth_t = base::depth_t; + using node_size_type = base::node_size_type; + using key_const_arg = base::key_const_arg; + using node_slot = base::node_slot; + using node_header = base::node_header; + using leaf_node = base::leaf_node; + using root_node = base::root_node; + using inner_node = base::inner_node; + using parent_node = base::parent_node; + using fwd_iterator = base::fwd_iterator; + using ra_iterator = base:: ra_iterator; + + using bptree_base::as; + using bptree_base::can_borrow; + using bptree_base::children; + using bptree_base::keys; + using bptree_base::node; + using bptree_base::num_chldrn; + using bptree_base::slot_of; + using bptree_base::underflowed; + using bptree_base::verify; + + using base::free; + using base::leaf_level; + using base::root; + +public: + using value_type = base::value_type; + using pointer = value_type *; + using const_pointer = value_type const *; + using reference = value_type &; + using const_reference = value_type const &; + using iterator = fwd_iterator; + using const_iterator = std::basic_const_iterator; + + using base::size; + using base::clear; + + static constexpr base::size_type max_size() noexcept + { + auto const max_number_of_nodes { std::numeric_limits::max() }; + auto const max_number_of_leaf_nodes{ /*TODO*/ max_number_of_nodes }; + auto const values_per_node { leaf_node::max_values }; + auto const max_sz { max_number_of_leaf_nodes * values_per_node }; + static_assert( max_sz <= std::numeric_limits::max() ); + return max_sz; + } + + [[ gnu::pure ]] iterator begin() noexcept { return { this->nodes_, this->first_leaf(), 0 }; } using stl_impl::begin; + [[ gnu::pure ]] iterator end () noexcept { return { this->nodes_, {} , 0 }; } using stl_impl::end ; + + [[ gnu::pure ]] ra_iterator ra_begin() noexcept { return { *this, this->first_leaf(), 0 }; } + [[ gnu::pure ]] ra_iterator ra_end () noexcept { return { *this, {} , size() }; } + [[ gnu::pure ]] std::basic_const_iterator ra_begin() const noexcept { return const_cast( *this ).ra_begin(); } + [[ gnu::pure ]] std::basic_const_iterator ra_end () const noexcept { return const_cast( *this ).ra_end (); } + + auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end() }; } + auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end() }; } + + void swap( bp_tree & other ) noexcept { base::swap( other ); } + + BOOST_NOINLINE + void insert( key_const_arg v ) + { + if ( base::empty() ) + { + auto & root{ static_cast( base::create_root() ) }; + BOOST_ASSUME( root.num_vals == 1 ); + root.keys[ 0 ] = v; + return; + } + + insert( find_nodes_for( v ).leaf, v, { /*insertion starts from leaves which do not have children*/ } ); + + ++this->hdr().size_; + } + + BOOST_NOINLINE + const_iterator find( key_const_arg key ) const noexcept { + auto const & leaf{ const_cast( *this ).find_nodes_for( key ).leaf }; + auto const pos{ find( leaf, key ) }; + if ( pos != leaf.num_vals && leaf.keys[ pos ] == key ) [[ likely ]] { + return iterator{ const_cast( *this ).nodes_, slot_of( leaf ), pos }; + } + + return this->cend(); + } + + BOOST_NOINLINE + void erase( key_const_arg key ) noexcept + { + auto & __restrict depth_{ base::hdr().depth_ }; + auto & __restrict root_ { base::hdr().root_ }; + auto & __restrict size_ { base::hdr().size_ }; + + auto const nodes{ find_nodes_for( key ) }; + + leaf_node & leaf{ nodes.leaf }; + if ( depth_ != 1 ) + verify( leaf ); + auto const key_offset{ find( leaf, key ) }; + + if ( nodes.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" + { + BOOST_ASSUME( key_offset == 0 ); + BOOST_ASSUME( leaf.keys[ key_offset ] == key ); + + auto * p_node{ &node( nodes.inner ) }; + auto & separator_key{ p_node->keys[ nodes.inner_offset ] }; + BOOST_ASSUME( separator_key == key ); + BOOST_ASSUME( key_offset + 1 < leaf.num_vals ); + separator_key = leaf.keys[ key_offset + 1 ]; + } + + + BOOST_ASSUME( key_offset != leaf.num_vals && leaf.keys[ key_offset ] == key ); + std::ranges::shift_left( keys( leaf ).subspan( key_offset ), 1 ); + BOOST_ASSUME( leaf.num_vals ); + --leaf.num_vals; + if ( depth_ == 1 ) [[ unlikely ]] // handle 'leaf root' deletion directly to simplify handle_underflow() + { + BOOST_ASSUME( root_ == slot_of( leaf ) ); + BOOST_ASSUME( leaf.is_root() ); + BOOST_ASSUME( size_ == leaf.num_vals + 1 ); + if ( leaf.num_vals == 0 ) { + root_ = {}; + base::free( leaf ); + --depth_; + } + } + else + if ( base::underflowed( leaf ) ) + { + BOOST_ASSUME( !leaf.is_root() ); + BOOST_ASSUME( depth_ > 1 ); + handle_underflow( leaf, base::leaf_level() ); + } + + --size_; + } + + Comparator const & comp() const noexcept { return *this; } + + // UB if the comparator is changed in such a way as to invalidate to order of elements already in the container + [[ nodiscard ]] Comparator & mutable_comp() noexcept { return *this; } + +private: + [[ gnu::pure, gnu::hot, clang::preserve_most, gnu::noinline ]] + auto find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept { + BOOST_ASSUME( num_vals > 0 ); + auto const posIter { std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp() ) }; + auto const posIndex{ std::distance( &keys[ 0 ], posIter ) }; + return static_cast( posIndex ); + } + auto find( auto const & node, key_const_arg value ) const noexcept { + return find( node.keys, node.num_vals, value ); + } + + template + void split_to_insert( N * p_node, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) { + verify( *p_node ); + + auto const max{ N::max_values }; // or 'order' + auto const mid{ max / 2 }; + + auto const node_slot{ slot_of( *p_node ) }; + auto * p_new_node{ &bptree_base::new_node() }; + auto const new_slot{ slot_of( *p_new_node ) }; + p_node = &bptree_base::node( node_slot ); // handle relocation + BOOST_ASSUME( p_node->num_vals == max ); + + auto constexpr parent_node_type{ requires{ p_new_node->children; } }; + + p_new_node->parent = p_node->parent; + if constexpr ( !parent_node_type ) + { + // leaf linked list + // p_node <- left/lesser | new_node -> right/greater + p_new_node->next = p_node->next; + p_node ->next = new_slot; + } + + auto const new_insert_pos { insert_pos - mid }; + auto const insertion_into_new_node{ insert_pos >= mid }; + Key key_to_propagate{ insertion_into_new_node + ? base::insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, new_insert_pos, key_right_child ) + : base::insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) + }; + + verify( *p_node ); + verify( *p_new_node ); + + if constexpr ( parent_node_type ) { + for ( auto const ch_slot : children( *p_new_node ) ) { + node( ch_slot ).parent = new_slot; + } + } + + // propagate the mid key to the parent + if ( p_node->is_root() ) [[ unlikely ]] { + auto & newRoot{ bptree_base::new_node() }; + auto & hdr { this->hdr() }; + p_node = &node( node_slot ); + p_new_node = &node( new_slot ); + newRoot.keys [ 0 ] = key_to_propagate; + newRoot.children[ 0 ] = node_slot; + newRoot.children[ 1 ] = new_slot; + newRoot.num_vals = 1; + hdr.root_ = slot_of( newRoot ); + p_node ->parent = hdr.root_; + p_new_node->parent = hdr.root_; + ++hdr.depth_; + } else { + return insert( node( p_node->parent ), key_to_propagate, new_slot ); + } + } + + template + void insert( N & target_node, key_const_arg v, node_slot const rightChild ) { + verify( target_node ); + auto const pos{ find( target_node, v ) }; + if ( base::full( target_node ) ) [[ unlikely ]] { + return split_to_insert( &target_node, pos, v, rightChild ); + } else { + ++target_node.num_vals; + std::ranges::shift_right( base::keys( target_node ).subspan( pos ), 1 ); + target_node.keys[ pos ] = v; + if constexpr ( requires { target_node.children; } ) { + std::ranges::shift_right( children( target_node ).subspan( pos + /*>right< child*/1 ), 1 ); + target_node.children[ pos + 1 ] = rightChild; + } + } + } + + + template + BOOST_NOINLINE + void handle_underflow( N & node, depth_t const level ) { + auto & __restrict depth_{ this->hdr().depth_ }; + auto & __restrict root_ { this->hdr().root_ }; + + BOOST_ASSUME( underflowed( node ) ); + + auto constexpr parent_node_type{ requires{ node.children; } }; + auto constexpr leaf_node_type { !parent_node_type }; + + if ( node.is_root() ) [[ unlikely ]] { + BOOST_ASSUME( level == 0 ); + BOOST_ASSUME( level < depth_ ); // 'leaf root' should be handled directly by the special case in erase() + BOOST_ASSUME( root_ == slot_of( node ) ); + if constexpr ( parent_node_type ) { + BOOST_ASSUME( !!node.children[ 0 ] ); + root_ = node.children[ 0 ]; + BOOST_ASSUME( depth_ > 1 ); + } + bptree_base::node( root_ ).parent = {}; + free( node ); + --depth_; + return; + } + + BOOST_ASSUME( level > 0 ); + if constexpr ( leaf_node_type ) { + BOOST_ASSUME( level == leaf_level() ); + } + auto const node_slot{ slot_of( node ) }; + auto & parent{ this->node( node.parent ) }; + verify( parent ); + + auto const parent_key_idx { find( parent, node.keys[ 0 ] ) }; BOOST_ASSUME( parent_key_idx != parent.num_vals || !leaf_node_type ); + auto const parent_has_key_copy{ leaf_node_type && ( parent.keys[ parent_key_idx ] == node.keys[ 0 ] ) }; + auto const parent_child_idx { parent_key_idx + parent_has_key_copy }; + BOOST_ASSUME( parent.children[ parent_child_idx ] == node_slot ); + + auto const right_separator_key_idx{ parent_key_idx + parent_has_key_copy }; + auto const left_separator_key_idx{ std::min( right_separator_key_idx - 1, parent.num_vals ) }; // (ab)use unsigned wraparound + auto const has_right_sibling { right_separator_key_idx != parent.num_vals }; + auto const has_left_sibling { left_separator_key_idx != parent.num_vals }; + auto const p_right_separator_key { has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; + auto const p_left_separator_key { has_left_sibling ? &parent.keys[ left_separator_key_idx ] : nullptr }; + BOOST_ASSUME( has_right_sibling == ( parent_child_idx < num_chldrn( parent ) - 1 ) ); + BOOST_ASSUME( has_left_sibling == ( parent_child_idx > 0 ) ); + auto const p_right_sibling{ has_right_sibling ? &this->node( children( parent )[ parent_child_idx + 1 ] ) : nullptr }; + auto const p_left_sibling { has_left_sibling ? &this->node( children( parent )[ parent_child_idx - 1 ] ) : nullptr }; + + BOOST_ASSERT( &node != p_left_sibling ); + BOOST_ASSERT( &node != p_right_sibling ); + BOOST_ASSERT( static_cast( &node ) != static_cast( &parent ) ); + // Borrow from left sibling if possible + if ( p_left_sibling && can_borrow( *p_left_sibling ) ) { + verify( *p_left_sibling ); + + node.num_vals++; + BOOST_ASSUME( node.num_vals <= node.max_values ); + + auto const node_vals{ keys( node ) }; + auto const left_vals{ keys( *p_left_sibling ) }; + std::ranges::shift_right( node_vals, 1 ); + + if constexpr ( leaf_node_type ) { + // Move the largest key from left sibling to the current node + BOOST_ASSUME( parent_has_key_copy ); + node_vals.front() = std::move( left_vals.back() ); + // adjust the separator key in the parent + BOOST_ASSERT( *p_left_separator_key == node_vals[ 1 ] ); + *p_left_separator_key = node_vals.front(); + } else { + // Move/rotate the largest key from left sibling to the current node 'through' the parent + BOOST_ASSERT( left_vals.back() < *p_left_separator_key ); + BOOST_ASSERT( *p_left_separator_key < node_vals.front() ); + node_vals.front() = std::move( *p_left_separator_key ); + *p_left_separator_key = std::move( left_vals.back() ); + + auto const chldrn{ children( node ) }; + std::ranges::shift_right( chldrn, 1 ); + chldrn.front() = std::move( children( *p_left_sibling ).back() ); + this->node( chldrn.front() ).parent = node_slot; + } + + p_left_sibling->num_vals--; + verify( *p_left_sibling ); + + BOOST_ASSUME( node. num_vals == N::min_values ); + BOOST_ASSUME( p_left_sibling->num_vals >= N::min_values ); + } + // Borrow from right sibling if possible + else if ( p_right_sibling && can_borrow( *p_right_sibling ) ) { + verify( *p_right_sibling ); + + node.num_vals++; + BOOST_ASSUME( node.num_vals <= node.max_values ); + + if constexpr ( leaf_node_type ) { + // Move the smallest key from right sibling to current node the parent + auto right_vals{ keys( *p_right_sibling ) }; + BOOST_ASSERT( parent.keys[ parent_child_idx ] == right_vals[ 0 ] ); + keys( node ).back() = std::move( right_vals.front() ); + std::ranges::shift_left( right_vals, 1 ); + // adjust the separator key in the parent + *p_right_separator_key = right_vals.front(); + } else { + // Move/rotate the smallest key from right sibling to current node 'through' the parent + BOOST_ASSUME( parent.keys[ parent_child_idx ] < p_right_sibling->keys[ 0 ] ); + BOOST_ASSERT( *( keys( node ).end() - 2 ) < *p_right_separator_key ); + keys( node ).back() = std::move( *p_right_separator_key ); + *p_right_separator_key = std::move( keys ( *p_right_sibling ).front() ); + children( node ).back() = std::move( children( *p_right_sibling ).front() ); + std::ranges::shift_left( keys ( *p_right_sibling ), 1 ); + std::ranges::shift_left( children( *p_right_sibling ), 1 ); + this->node( children( node ).back() ).parent = node_slot; + } + + p_right_sibling->num_vals--; + verify( *p_right_sibling ); + + BOOST_ASSUME( node. num_vals == N::min_values ); + BOOST_ASSUME( p_right_sibling->num_vals >= N::min_values ); + } + // Merge with left or right sibling + else { + if ( p_left_sibling ) { + BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); + // Merge node -> left sibling + this->merge_nodes( node, *p_left_sibling, parent, left_separator_key_idx, parent_child_idx ); + } else { + BOOST_ASSUME( parent_key_idx == 0 ); + BOOST_ASSUME( parent.keys[ parent_key_idx ] <= p_right_sibling->keys[ 0 ] ); + // Merge right sibling -> node + this->merge_nodes( *p_right_sibling, node, parent, right_separator_key_idx, parent_child_idx + 1 ); + } + + // propagate underflow + if ( + ( ( level == 1 ) && underflowed( as( parent ) ) ) || + ( ( level != 1 ) && underflowed( parent ) ) + ) + { + BOOST_ASSUME( level > 0 ); + BOOST_ASSUME( level < depth_ ); + handle_underflow( parent, level - 1 ); + } + } + } + + + struct key_locations + { + leaf_node & leaf; + // optional - if also present in an inner node as a separator key + node_slot inner; + node_size_type inner_offset; + }; + + [[ gnu::hot, gnu::sysv_abi ]] + key_locations find_nodes_for( key_const_arg key ) noexcept + { + node_slot separator_key_node; + node_size_type separator_key_offset; + // if the root is a (lone) leaf_node is implicitly handled by the loop condition: + // depth_ == 1 so the loop is skipped entirely and the lone root is never examined + // through the incorrectly typed reference + auto * p_node{ &bptree_base::as( root() ) }; + for ( auto level{ 0 }; level < this->hdr().depth_ - 1; ++level ) + { + auto pos{ find( *p_node, key ) }; + if ( pos != p_node->num_vals && p_node->keys[ pos ] == key ) { + // separator key - it also means we have to traverse to the right + BOOST_ASSUME( !separator_key_node ); + separator_key_node = slot_of( *p_node ); + separator_key_offset = pos; + ++pos; // traverse to the right child + } + p_node = &node( p_node->children[ pos ] ); + } + return { base::template as( *p_node ), separator_key_node, separator_key_offset }; + } +}; // class bp_tree + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/b+tree_print.hpp b/include/psi/vm/containers/b+tree_print.hpp new file mode 100644 index 0000000..5940635 --- /dev/null +++ b/include/psi/vm/containers/b+tree_print.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include "b+tree.hpp" + +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +template +void bptree_base_wkey::print() const +{ + if ( empty() ) + { + std::puts( "The tree is empty." ); + return; + } + + // Queue for performing level-order traversal (BFS). + std::queue nodes; + nodes.push( &as( root() ) ); + + // Current level tracker + depth_t level{ 0 }; + + // Perform BFS, processing one level of the tree at a time. + while ( !nodes.empty() ) + { + auto node_count{ nodes.size() }; + std::print( "Level {} ({} nodes):\n\t", std::uint16_t( level ), node_count ); + + size_t level_key_count{ 0 }; + // Process all nodes at the current level + while ( node_count > 0 ) + { + auto const node{ nodes.front() }; + nodes.pop(); + + if ( is_leaf_level( level ) ) + { + auto & ln{ as( *node ) }; + level_key_count += num_vals( ln ); + std::putchar( '[' ); + for ( auto i{ 0 }; i < num_vals( ln ); ++i ) + { + std::print( "{}", keys( ln )[ i ] ); + if ( i < num_vals( ln ) - 1 ) + std::print( ", " ); + } + std::print( "] " ); + } + else + { + // Internal node, print keys and add children to the queue + level_key_count += num_vals( *node ); + std::putchar( '<' ); + for ( auto i{ 0 }; i < num_vals( *node ); ++i ) + { + std::print( "{}", keys( *node )[ i ] ); + if ( i < num_vals( *node ) - 1 ) + std::print( ", " ); + } + std::print( "> " ); + + // Add all children of this internal node to the queue + for ( auto i{ 0 }; i < num_chldrn( *node ); ++i ) + nodes.push( &this->node( node->children[ i ] ) ); + } + + --node_count; + } + + std::println( " [{} values]", level_key_count ); + ++level; + } +} + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index 20b2301..d7e3251 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -352,12 +352,14 @@ struct value_init_t {}; inline constexpr value_init_t value_init ; struct header_info { + using align_t = std::uint16_t; // fit page_size + constexpr header_info() = default; - constexpr header_info( std::uint32_t const size, std::uint8_t const alignment ) noexcept : header_size{ size }, data_extra_alignment{ alignment } {} + constexpr header_info( std::uint32_t const size, align_t const alignment ) noexcept : header_size{ size }, data_extra_alignment{ alignment } {} template - constexpr header_info( std::in_place_type_t, std::uint8_t const extra_alignment = 1 ) noexcept : header_info{ sizeof( T ), extra_alignment } {} + constexpr header_info( std::in_place_type_t, align_t const extra_alignment = 1 ) noexcept : header_info{ sizeof( T ), extra_alignment } {} template - static constexpr header_info make( std::uint8_t const extra_alignment = 1 ) noexcept { return { sizeof( T ), extra_alignment }; } + static constexpr header_info make( align_t const extra_alignment = 1 ) noexcept { return { sizeof( T ), extra_alignment }; } template header_info add_header() const noexcept // support chained headers (class hierarchies) @@ -374,13 +376,13 @@ struct header_info header_info with_alignment_for() const noexcept { BOOST_ASSUME( data_extra_alignment >= 1 ); - return { this->header_size, std::max({ this->data_extra_alignment, alignof( T )... }) }; + return { this->header_size, std::max({ this->data_extra_alignment, alignof( T )... }) }; } std::uint32_t final_header_size() const noexcept { return align_up( header_size, data_extra_alignment ); } std::uint32_t header_size { 0 }; - std::uint8_t data_extra_alignment{ 1 }; // e.g. for vectorization or overlaying complex types over std::byte storage + align_t data_extra_alignment{ 1 }; // e.g. for vectorization or overlaying complex types over std::byte storage }; // header_info // Standard-vector-like/compatible _presistent_ container class template @@ -1038,7 +1040,7 @@ class vector //! Throws: Nothing. //! //! Complexity: Constant. - void swap( vector & x ) noexcept { swap( this->storage, x.storage ); } + void swap( vector & x ) noexcept { using std::swap; swap( this->storage_, x.storage_ ); } //! Effects: Erases all the elements of the vector. //! diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp new file mode 100644 index 0000000..5734d46 --- /dev/null +++ b/src/containers/b+tree.cpp @@ -0,0 +1,171 @@ +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +// https://en.wikipedia.org/wiki/B-tree +// https://opendsa-server.cs.vt.edu/ODSA/Books/CS3/html/BTree.html +// http://www.cburch.com/cs/340/reading/btree/index.html +// https://www.programiz.com/dsa/b-plus-tree +// https://www.geeksforgeeks.org/b-trees-implementation-in-c +// https://github.com/jeffplaisance/BppTree +// https://flatcap.github.io/linux-ntfs/ntfs/concepts/tree/index.html +// https://benjamincongdon.me/blog/2021/08/17/B-Trees-More-Than-I-Thought-Id-Want-to-Know +// https://stackoverflow.com/questions/59362113/b-tree-minimum-internal-children-count-explanation +// https://web.archive.org/web/20190126073810/http://supertech.csail.mit.edu/cacheObliviousBTree.html +// https://www.researchgate.net/publication/220225482_Cache-Oblivious_Databases_Limitations_and_Opportunities +// Data Structure Visualizations https://www.cs.usfca.edu/~galles/visualization/Algorithms.html +// Griffin: Fast Transactional Database Index with Hash and B+-Tree https://ieeexplore.ieee.org/abstract/document/10678674 +// Restructuring the concurrent B+-tree with non-blocked search operations https://www.sciencedirect.com/science/article/abs/pii/S002002550200261X + +bptree_base::bptree_base( header_info const hdr_info ) noexcept + : + nodes_{ hdr_info.add_header
() } +{} + +void bptree_base::clear() noexcept +{ + nodes_.clear(); + hdr() = {}; +} + +std::span bptree_base::user_header_data() noexcept { return header_data().second; } + +bptree_base::header & bptree_base::hdr() noexcept { return *header_data().first; } + +bptree_base::storage_result bptree_base::init_header( storage_result success ) noexcept +{ + if ( std::move( success ) && nodes_.empty() ) + hdr() = {}; + return success; +} + +bptree_base::base_iterator::base_iterator( node_pool & nodes, node_slot const node_offset, node_size_type const value_offset ) noexcept + : +#ifndef NDEBUG // for bounds checking + nodes_{ nodes }, +#else + nodes_{ nodes.data() }, +#endif + node_slot_{ node_offset }, value_offset_{ value_offset } +{} + +bptree_base::linked_node_header & +bptree_base::base_iterator::node() const noexcept { return static_cast( static_cast( nodes_[ *node_slot_ ] ) ); } + +bptree_base::base_iterator & +bptree_base::base_iterator::operator++() noexcept +{ + BOOST_ASSERT_MSG( node_slot_, "Iterator at end: not incrementable" ); + auto & node{ this->node() }; + BOOST_ASSUME( node.num_vals >= 1 ); + if ( ++value_offset_ == node.num_vals ) [[ unlikely ]] { + // implicitly becomes an end iterator when node.next is 'null' + node_slot_ = node.next; + value_offset_ = 0; + } + return *this; +} + +bool bptree_base::base_iterator::operator==( base_iterator const & other ) const noexcept +{ + BOOST_ASSERT_MSG( &this->nodes_[ 0 ] == &other.nodes_[ 0 ], "Comparing iterators from different containers" ); + return ( this->node_slot_ == other.node_slot_ ) && ( this->value_offset_ == other.value_offset_ ); +} + + +bptree_base::base_random_access_iterator & +bptree_base::base_random_access_iterator::operator+=( difference_type const n ) noexcept +{ + if ( n < 0 ) + { + BOOST_ASSUME( static_cast( -n ) < index_ ); + auto const absolute_pos{ index_ + n }; + node_slot_ = leaves_start_; + value_offset_ = 0; + index_ = 0; + return *this += absolute_pos; + } + + BOOST_ASSERT_MSG( node_slot_, "Iterator at end: not incrementable" ); + auto un{ static_cast( n ) }; + for ( ;; ) { + auto & node{ this->node() }; + auto const available_offset{ static_cast( node.num_vals - 1 - value_offset_ ) }; + if ( available_offset >= un ) [[ likely ]] { + value_offset_ += static_cast( un ); + break; + } else { + un -= available_offset; + node_slot_ = node.next; + value_offset_ = 0; + BOOST_ASSERT_MSG( node_slot_ || un == 0, "Incrementing out of bounds" ); + } + } + index_ += static_cast( n ); + return *this; +} + + +void bptree_base::swap( bptree_base & other ) noexcept +{ + using std::swap; + swap( this->nodes_, other.nodes_ ); +} + +[[ gnu::cold ]] +bptree_base::linked_node_header & +bptree_base::create_root() +{ + BOOST_ASSUME( !hdr().root_ ); + BOOST_ASSUME( !hdr().depth_ ); + BOOST_ASSUME( !hdr().size_ ); + // the initial/'lone' root node is a leaf node + auto & root{ static_cast( static_cast( new_node() ) ) }; + auto & hdr { this->hdr() }; + BOOST_ASSUME( root.num_vals == 0 ); + root.num_vals = 1; + root.next = {}; + hdr.root_ = slot_of( root ); + hdr.leaves_ = hdr.root_; + hdr.depth_ = 1; + hdr.size_ = 1; + return root; +} + +bptree_base::depth_t bptree_base:: leaf_level() const noexcept { BOOST_ASSUME( hdr().depth_ ); return hdr().depth_ - 1; } + bool bptree_base::is_leaf_level( depth_t const level ) const noexcept { return level == leaf_level(); } + +void bptree_base::free( node_header & node ) noexcept +{ + auto & free_list{ hdr().free_list_ }; + auto & free_node{ static_cast( node ) }; + if ( free_list ) + free_node.next = free_list; + free_list = slot_of( free_node ); +} + +bptree_base::node_slot +bptree_base::slot_of( node_header const & node ) const noexcept +{ + return { static_cast( static_cast( &node ) - nodes_.data() ) }; +} + +[[ gnu::noinline ]] +bptree_base::node_placeholder & +bptree_base::new_node() +{ + auto & free_list{ hdr().free_list_ }; + if ( free_list ) + { + auto & cached_node{ node( free_list ) }; + free_list = cached_node.next; + return as( cached_node ); + } + return nodes_.emplace_back(); +} + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/src/pch.hpp b/src/pch.hpp new file mode 100644 index 0000000..ad07223 --- /dev/null +++ b/src/pch.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#undef ERROR +#endif \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8f996c7..ef79c08 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,16 +1,18 @@ # psi::vm tests # Fix ODR violations and compiler and linker option mismatches: add GTest after everything is already setup. -CPMAddPackage( "gh:google/googletest@1.14.0" ) +CPMAddPackage( "gh:google/googletest@1.15.2" ) include( GoogleTest ) set( vm_test_sources + "${CMAKE_CURRENT_SOURCE_DIR}/b+tree.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/vector.cpp" ) add_executable( vm_unit_tests EXCLUDE_FROM_ALL ${vm_test_sources} ) target_link_libraries( vm_unit_tests PRIVATE GTest::gtest_main psi::vm ) +target_precompile_headers( vm_unit_tests REUSE_FROM psi_vm ) set_target_properties( vm_unit_tests diff --git a/test/b+tree.cpp b/test/b+tree.cpp new file mode 100644 index 0000000..4aa574e --- /dev/null +++ b/test/b+tree.cpp @@ -0,0 +1,59 @@ +#include +#include + +#include + +#include + +#include +#include +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +static auto const test_file{ "test.bpt" }; + +TEST( bp_tree, playground ) +{ + std::ranges::iota_view constexpr sorted_numbers{ 0, 913735 }; + std::mt19937 rng{ std::random_device{}() }; + auto numbers{ std::ranges::to( sorted_numbers ) }; + std::shuffle( numbers.begin(), numbers.end(), rng ); + + { + bp_tree bpt; + bpt.map_file( test_file, flags::named_object_construction_policy::create_new_or_truncate_existing ); + + for ( auto const & n : numbers ) + bpt.insert( n ); + + EXPECT_TRUE( std::ranges::is_sorted( bpt ) ); + EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); + EXPECT_NE( bpt.find( +42 ), bpt.end() ); + EXPECT_EQ( bpt.find( -42 ), bpt.end() ); + bpt.erase( 42 ); + EXPECT_EQ( bpt.find( +42 ), bpt.end() ); + } + { + bp_tree bpt; + bpt.map_file( test_file, flags::named_object_construction_policy::open_existing ); + + EXPECT_EQ( bpt.size(), sorted_numbers.size() - 1 ); + bpt.insert( +42 ); + + EXPECT_TRUE( std::ranges::is_sorted( bpt ) ); + EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); + EXPECT_NE( bpt.find( +42 ), bpt.end() ); + EXPECT_EQ( bpt.find( -42 ), bpt.end() ); + + bpt.clear(); + bpt.print(); + } +} + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ From 7924f16a33ea9cdf841ed1b4fc8ada9159c1736c Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 00:25:49 +0200 Subject: [PATCH 03/49] Fixed a bug in insert_into_new_node() and insert_into_existing_node() overloads for parent_nodes. Fixes for various compilation warnings and errors. Minor cleanups. --- include/psi/vm/containers/b+tree.hpp | 74 ++++++++++++------- include/psi/vm/containers/b+tree_print.hpp | 4 +- .../shared_memory/android/mem.hpp | 2 +- include/psi/vm/vector.hpp | 4 +- src/containers/b+tree.cpp | 9 ++- src/detail/nt.cpp | 5 ++ .../shared_memory/mem.posix.cpp | 2 +- test/CMakeLists.txt | 4 +- test/b+tree.cpp | 7 +- 9 files changed, 72 insertions(+), 39 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index a8ac721..cedb5f0 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -4,6 +4,8 @@ #include #include +#include + #include #include #include @@ -14,13 +16,26 @@ #include #include #include +#include +#include #include #include //------------------------------------------------------------------------------ +#if __cpp_lib_shift < 202202L +namespace std::ranges +{ + auto shift_left ( auto && rng, auto d ) noexcept { return std::shift_left ( rng.begin(), rng.end(), d ); } + auto shift_right( auto && rng, auto d ) noexcept { return std::shift_right( rng.begin(), rng.end(), d ); } +} +#endif +//------------------------------------------------------------------------------ namespace psi::vm { //------------------------------------------------------------------------------ +PSI_WARNING_DISABLE_PUSH() +PSI_WARNING_MSVC_DISABLE( 5030 ) // unrecognized attribute + namespace detail { template @@ -54,8 +69,8 @@ class bptree_base bool has_attached_storage() const noexcept { return nodes_.has_attached_storage(); } - storage_result map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { return init_header( nodes_.map_file ( file, policy ) ); } - storage_result map_memory( size_type const initial_capacity_in_nodes = 0 ) noexcept { return init_header( nodes_.map_memory( initial_capacity_in_nodes ) ); } + storage_result map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { return init_header( nodes_.map_file ( file, policy ) ); } + storage_result map_memory( std::uint32_t const initial_capacity_as_number_of_nodes = 0 ) noexcept { return init_header( nodes_.map_memory( initial_capacity_as_number_of_nodes ) ); } protected: static constexpr auto node_size{ page_size }; @@ -258,6 +273,8 @@ class bptree_base::base_random_access_iterator : public base_iterator // should implicitly handle end iterator comparison also (this requires the start_index constructor argument for the construction of end iterators) friend constexpr bool operator==( base_random_access_iterator const & left, base_random_access_iterator const & right ) noexcept { return left.index_ == right.index_; } + + auto operator<=>( base_random_access_iterator const & other ) const noexcept { return this->index_ <=> other.index_; } }; // class base_random_access_iterator @@ -357,13 +374,18 @@ class bptree_base_wkey : public bptree_base auto const max{ N::max_values }; // or 'order' auto const mid{ max / 2 }; + new_node.num_vals = max - mid; + value_type key_to_propagate; - if ( new_insert_pos == 0 ) { + if ( new_insert_pos == 0 ) + { umove<&N::keys >( node, mid , node.num_vals , new_node, 0 ); umove<&N::children>( node, mid + 1, node.num_vals + 1, new_node, 1 ); children( new_node )[ new_insert_pos ] = key_right_child; key_to_propagate = std::move( value ); - } else { + } + else + { key_to_propagate = std::move( node.keys[ mid ] ); umove<&N::keys >( node, mid + 1, insert_pos , new_node, 0 ); @@ -376,8 +398,7 @@ class bptree_base_wkey : public bptree_base children( new_node )[ new_insert_pos ] = key_right_child; } - node .num_vals = mid ; - new_node.num_vals = max - mid; + node.num_vals = mid; return key_to_propagate; } @@ -424,12 +445,12 @@ class bptree_base_wkey : public bptree_base umove<&N::keys >( node, insert_pos , mid - 1, node, insert_pos + 1 ); umove<&N::children>( node, insert_pos + 1, mid , node, insert_pos + 2 ); - keys ( node )[ insert_pos ] = value; - children( node )[ insert_pos + 1 ] = key_right_child; - node .num_vals = mid; new_node.num_vals = max - mid; + keys ( node )[ insert_pos ] = value; + children( node )[ insert_pos + 1 ] = key_right_child; + return key_to_propagate; } @@ -481,7 +502,6 @@ class bptree_base_wkey : public bptree_base source.num_vals = 0; target.next = source.next; BOOST_ASSUME( target.num_vals == target.max_values - 1 ); - std::ranges::shift_left( keys ( parent ).subspan( parent_key_idx ), 1 ); std::ranges::shift_left( children( parent ).subspan( parent_child_idx ), 1 ); BOOST_ASSUME( parent.num_vals ); @@ -744,13 +764,15 @@ class bp_tree private: [[ gnu::pure, gnu::hot, clang::preserve_most, gnu::noinline ]] - auto find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept { + auto find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept + { BOOST_ASSUME( num_vals > 0 ); - auto const posIter { std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp() ) }; - auto const posIndex{ std::distance( &keys[ 0 ], posIter ) }; - return static_cast( posIndex ); + auto const pos_iter{ std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp() ) }; + auto const pos_idx { std::distance( &keys[ 0 ], pos_iter ) }; + return static_cast( pos_idx ); } - auto find( auto const & node, key_const_arg value ) const noexcept { + auto find( auto const & node, key_const_arg value ) const noexcept + { return find( node.keys, node.num_vals, value ); } @@ -758,8 +780,8 @@ class bp_tree void split_to_insert( N * p_node, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) { verify( *p_node ); - auto const max{ N::max_values }; // or 'order' - auto const mid{ max / 2 }; + node_size_type const max{ N::max_values }; // or 'order' + node_size_type const mid{ max / 2 }; auto const node_slot{ slot_of( *p_node ) }; auto * p_new_node{ &bptree_base::new_node() }; @@ -778,8 +800,8 @@ class bp_tree p_node ->next = new_slot; } - auto const new_insert_pos { insert_pos - mid }; - auto const insertion_into_new_node{ insert_pos >= mid }; + node_size_type const new_insert_pos ( insert_pos - mid ); + bool const insertion_into_new_node{ insert_pos >= mid }; Key key_to_propagate{ insertion_into_new_node ? base::insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, new_insert_pos, key_right_child ) : base::insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) @@ -866,12 +888,12 @@ class bp_tree verify( parent ); auto const parent_key_idx { find( parent, node.keys[ 0 ] ) }; BOOST_ASSUME( parent_key_idx != parent.num_vals || !leaf_node_type ); - auto const parent_has_key_copy{ leaf_node_type && ( parent.keys[ parent_key_idx ] == node.keys[ 0 ] ) }; - auto const parent_child_idx { parent_key_idx + parent_has_key_copy }; + bool const parent_has_key_copy{ leaf_node_type && ( parent.keys[ parent_key_idx ] == node.keys[ 0 ] ) }; + auto const parent_child_idx { static_cast( parent_key_idx + parent_has_key_copy ) }; BOOST_ASSUME( parent.children[ parent_child_idx ] == node_slot ); - auto const right_separator_key_idx{ parent_key_idx + parent_has_key_copy }; - auto const left_separator_key_idx{ std::min( right_separator_key_idx - 1, parent.num_vals ) }; // (ab)use unsigned wraparound + auto const right_separator_key_idx{ static_cast( parent_key_idx + parent_has_key_copy ) }; + auto const left_separator_key_idx{ std::min( static_cast( right_separator_key_idx - 1 ), parent.num_vals ) }; // (ab)use unsigned wraparound auto const has_right_sibling { right_separator_key_idx != parent.num_vals }; auto const has_left_sibling { left_separator_key_idx != parent.num_vals }; auto const p_right_separator_key { has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; @@ -964,7 +986,7 @@ class bp_tree BOOST_ASSUME( parent_key_idx == 0 ); BOOST_ASSUME( parent.keys[ parent_key_idx ] <= p_right_sibling->keys[ 0 ] ); // Merge right sibling -> node - this->merge_nodes( *p_right_sibling, node, parent, right_separator_key_idx, parent_child_idx + 1 ); + this->merge_nodes( *p_right_sibling, node, parent, right_separator_key_idx, static_cast( parent_child_idx + 1 ) ); } // propagate underflow @@ -993,7 +1015,7 @@ class bp_tree key_locations find_nodes_for( key_const_arg key ) noexcept { node_slot separator_key_node; - node_size_type separator_key_offset; + node_size_type separator_key_offset{}; // if the root is a (lone) leaf_node is implicitly handled by the loop condition: // depth_ == 1 so the loop is skipped entirely and the lone root is never examined // through the incorrectly typed reference @@ -1014,6 +1036,8 @@ class bp_tree } }; // class bp_tree +PSI_WARNING_DISABLE_POP() + //------------------------------------------------------------------------------ } // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/b+tree_print.hpp b/include/psi/vm/containers/b+tree_print.hpp index 5940635..d1345e5 100644 --- a/include/psi/vm/containers/b+tree_print.hpp +++ b/include/psi/vm/containers/b+tree_print.hpp @@ -43,7 +43,7 @@ void bptree_base_wkey::print() const auto & ln{ as( *node ) }; level_key_count += num_vals( ln ); std::putchar( '[' ); - for ( auto i{ 0 }; i < num_vals( ln ); ++i ) + for ( auto i{ 0U }; i < num_vals( ln ); ++i ) { std::print( "{}", keys( ln )[ i ] ); if ( i < num_vals( ln ) - 1 ) @@ -56,7 +56,7 @@ void bptree_base_wkey::print() const // Internal node, print keys and add children to the queue level_key_count += num_vals( *node ); std::putchar( '<' ); - for ( auto i{ 0 }; i < num_vals( *node ); ++i ) + for ( auto i{ 0U }; i < num_vals( *node ); ++i ) { std::print( "{}", keys( *node )[ i ] ); if ( i < num_vals( *node ) - 1 ) diff --git a/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp b/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp index 489fc24..4c21a7e 100644 --- a/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp +++ b/include/psi/vm/mappable_objects/shared_memory/android/mem.hpp @@ -69,7 +69,7 @@ namespace detail std::memcpy( prefixed_name + shm_prefix.size(), name, name_length + 1 ); } - BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_WARN_UNUSED_RESULT ) + BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS ) [[ nodiscard ]] file_handle::reference BOOST_CC_REG shm_open ( char const * const name, diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index d7e3251..d85c4f4 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -329,7 +329,7 @@ class contiguous_container_storage [[ gnu::pure, nodiscard ]] size_type & stored_size() noexcept requires( !headerless ) { auto const p_size{ contiguous_container_storage_base::data() + header_size() - size_size }; - BOOST_ASSERT( reinterpret_cast( p_size ) % alignof( size_type ) == 0 ); + BOOST_ASSERT( reinterpret_cast( p_size ) % alignof( size_type ) == 0 ); return *reinterpret_cast( p_size ); } [[ gnu::pure, nodiscard ]] size_type stored_size() const noexcept requires( !headerless ) @@ -1180,8 +1180,6 @@ class vector iterator make_space_for_insert( const_iterator const position, size_type const n ) { - using ssize_type = std::make_signed_t; - verify_iterator( position ); auto const position_index{ index_of( position ) }; auto const current_size { size() }; diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index 5734d46..076231f 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -80,12 +80,13 @@ bptree_base::base_random_access_iterator::operator+=( difference_type const n ) { if ( n < 0 ) { - BOOST_ASSUME( static_cast( -n ) < index_ ); - auto const absolute_pos{ index_ + n }; - node_slot_ = leaves_start_; + auto const un{ static_cast( -n ) }; + BOOST_ASSUME( un < index_ ); + auto const absolute_pos{ index_ - un }; + node_slot_ = leaves_start_; value_offset_ = 0; index_ = 0; - return *this += absolute_pos; + return *this += static_cast( absolute_pos ); } BOOST_ASSERT_MSG( node_slot_, "Iterator at end: not incrementable" ); diff --git a/src/detail/nt.cpp b/src/detail/nt.cpp index bc691e9..fbfea22 100644 --- a/src/detail/nt.cpp +++ b/src/detail/nt.cpp @@ -21,6 +21,8 @@ //------------------------------------------------------------------------------ #include +#include + #include #pragma comment( lib, "ntdll.lib" ) @@ -37,8 +39,11 @@ namespace detail [[ gnu::constructor( 101 ) ]] void init_ntdll_handle() noexcept { ntdll = ::GetModuleHandleW( L"ntdll.dll" ); } #else + PSI_WARNING_DISABLE_PUSH() + PSI_WARNING_MSVC_DISABLE( 4073 ) // initializers put in library initialization area #pragma init_seg( lib ) HMODULE const ntdll{ ::GetModuleHandleW( L"ntdll.dll" ) }; + PSI_WARNING_DISABLE_POP() #endif BOOST_ATTRIBUTES( BOOST_COLD, BOOST_RESTRICTED_FUNCTION_L3, BOOST_RESTRICTED_FUNCTION_RETURN ) diff --git a/src/mappable_objects/shared_memory/mem.posix.cpp b/src/mappable_objects/shared_memory/mem.posix.cpp index 772f92e..b2dd85c 100644 --- a/src/mappable_objects/shared_memory/mem.posix.cpp +++ b/src/mappable_objects/shared_memory/mem.posix.cpp @@ -234,7 +234,7 @@ namespace detail return result; } - BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS, BOOST_WARN_UNUSED_RESULT ) + [[ nodiscard ]] BOOST_ATTRIBUTES( BOOST_MINSIZE, BOOST_EXCEPTIONLESS ) bool named_semaphore::semop( short const opcode, bool const nowait /*= false*/ ) noexcept { // http://linux.die.net/man/2/semop diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ef79c08..af92336 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -12,7 +12,9 @@ set( vm_test_sources add_executable( vm_unit_tests EXCLUDE_FROM_ALL ${vm_test_sources} ) target_link_libraries( vm_unit_tests PRIVATE GTest::gtest_main psi::vm ) -target_precompile_headers( vm_unit_tests REUSE_FROM psi_vm ) +if ( NOT MSVC OR CLANG_CL ) #:wat: msvc fails to find the pch file in release builds!? + target_precompile_headers( vm_unit_tests REUSE_FROM psi_vm ) +endif() set_target_properties( vm_unit_tests diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 4aa574e..3ab42f4 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include //------------------------------------------------------------------------------ namespace psi::vm @@ -30,7 +31,9 @@ TEST( bp_tree, playground ) for ( auto const & n : numbers ) bpt.insert( n ); - EXPECT_TRUE( std::ranges::is_sorted( bpt ) ); + static_assert( std::forward_iterator::const_iterator> ); + + EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ) ) ); EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); EXPECT_NE( bpt.find( +42 ), bpt.end() ); EXPECT_EQ( bpt.find( -42 ), bpt.end() ); @@ -45,7 +48,7 @@ TEST( bp_tree, playground ) bpt.insert( +42 ); EXPECT_TRUE( std::ranges::is_sorted( bpt ) ); - EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); + EXPECT_TRUE( std::ranges::equal( std::as_const( bpt ), sorted_numbers ) ); EXPECT_NE( bpt.find( +42 ), bpt.end() ); EXPECT_EQ( bpt.find( -42 ), bpt.end() ); From 15ef1f9e333a478773a2122a35478362210cfae9 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 03:03:03 +0200 Subject: [PATCH 04/49] Fixed bugs in both overloads of insert_into_existing_node(). Added more sanity checks. Minor cleanups. --- include/psi/vm/containers/b+tree.hpp | 36 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index cedb5f0..7b0a949 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -374,6 +374,9 @@ class bptree_base_wkey : public bptree_base auto const max{ N::max_values }; // or 'order' auto const mid{ max / 2 }; + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + new_node.num_vals = max - mid; value_type key_to_propagate; @@ -417,8 +420,11 @@ class bptree_base_wkey : public bptree_base auto const max{ N::max_values }; // or 'order' auto const mid{ max / 2 }; - umove<&N::keys>( node, mid , insert_pos , new_node, 0 ); - umove<&N::keys>( node, insert_pos, node.num_vals, new_node, new_insert_pos + 1 ); + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + + umove<&N::keys>( node, mid , insert_pos, new_node, 0 ); + umove<&N::keys>( node, insert_pos, max , new_node, new_insert_pos + 1 ); node .num_vals = mid ; new_node.num_vals = max - mid + 1; @@ -437,13 +443,16 @@ class bptree_base_wkey : public bptree_base auto const max{ N::max_values }; // or 'order' auto const mid{ max / 2 }; + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + value_type key_to_propagate{ std::move( node.keys[ mid - 1 ] ) }; umove<&N::keys >( node, mid, node.num_vals , new_node, 0 ); umove<&N::children>( node, mid, node.num_vals + 1, new_node, 0 ); - umove<&N::keys >( node, insert_pos , mid - 1, node, insert_pos + 1 ); - umove<&N::children>( node, insert_pos + 1, mid , node, insert_pos + 2 ); + std::shift_right( &node.keys [ insert_pos ], &node.keys [ mid ], 1 ); + std::shift_right( &node.children[ insert_pos + 1 ], &node.children[ mid + 1 ], 1 ); node .num_vals = mid; new_node.num_vals = max - mid; @@ -463,8 +472,11 @@ class bptree_base_wkey : public bptree_base auto const max{ N::max_values }; // or 'order' auto const mid{ max / 2 }; - umove<&N::keys>( node, mid , node.num_vals, new_node, 0 ); - umove<&N::keys>( node, insert_pos, mid , node , insert_pos + 1 ); + BOOST_ASSUME( node.num_vals == max ); + BOOST_ASSUME( new_node.num_vals == 0 ); + + umove<&N::keys> ( node, mid, max, new_node, 0 ); + std::shift_right( &node.keys[ insert_pos ], &node.keys[ mid + 1 ], 1 ); node .num_vals = mid + 1 ; new_node.num_vals = max - mid; @@ -800,11 +812,11 @@ class bp_tree p_node ->next = new_slot; } - node_size_type const new_insert_pos ( insert_pos - mid ); - bool const insertion_into_new_node{ insert_pos >= mid }; - Key key_to_propagate{ insertion_into_new_node - ? base::insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, new_insert_pos, key_right_child ) - : base::insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) + auto const new_insert_pos { insert_pos - mid }; + bool const insertion_into_new_node{ insert_pos >= mid }; + decltype( auto ) key_to_propagate{ insertion_into_new_node + ? base::insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) + : base::insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) }; verify( *p_node ); @@ -822,7 +834,7 @@ class bp_tree auto & hdr { this->hdr() }; p_node = &node( node_slot ); p_new_node = &node( new_slot ); - newRoot.keys [ 0 ] = key_to_propagate; + newRoot.keys [ 0 ] = std::move( key_to_propagate ); newRoot.children[ 0 ] = node_slot; newRoot.children[ 1 ] = new_slot; newRoot.num_vals = 1; From 5d5a3c21c8a95681bd535247739a6c98759b16f9 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 03:03:31 +0200 Subject: [PATCH 05/49] Shortened b+tree debug test duration. --- test/b+tree.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 3ab42f4..96c34ed 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -19,7 +19,12 @@ static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) { - std::ranges::iota_view constexpr sorted_numbers{ 0, 913735 }; +#ifdef NDEBUG + auto const test_size{ 913735 }; +#else + auto const test_size{ 93735 }; +#endif + std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; std::mt19937 rng{ std::random_device{}() }; auto numbers{ std::ranges::to( sorted_numbers ) }; std::shuffle( numbers.begin(), numbers.end(), rng ); From f3a4c78c9bde5aae42b584c9d64b97ce09426220 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 03:31:55 +0200 Subject: [PATCH 06/49] Compiler warning fix. --- include/psi/vm/containers/b+tree_print.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/psi/vm/containers/b+tree_print.hpp b/include/psi/vm/containers/b+tree_print.hpp index d1345e5..5d65ab3 100644 --- a/include/psi/vm/containers/b+tree_print.hpp +++ b/include/psi/vm/containers/b+tree_print.hpp @@ -46,7 +46,7 @@ void bptree_base_wkey::print() const for ( auto i{ 0U }; i < num_vals( ln ); ++i ) { std::print( "{}", keys( ln )[ i ] ); - if ( i < num_vals( ln ) - 1 ) + if ( i < num_vals( ln ) - 1U ) std::print( ", " ); } std::print( "] " ); @@ -59,7 +59,7 @@ void bptree_base_wkey::print() const for ( auto i{ 0U }; i < num_vals( *node ); ++i ) { std::print( "{}", keys( *node )[ i ] ); - if ( i < num_vals( *node ) - 1 ) + if ( i < num_vals( *node ) - 1U ) std::print( ", " ); } std::print( "> " ); From 10ef0f96dfb7e04ff7c0865af7d1cb1bd441071b Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 03:34:47 +0200 Subject: [PATCH 07/49] Fixed a bug in split_to_insert(). --- include/psi/vm/containers/b+tree.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 7b0a949..283043b 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -814,7 +814,7 @@ class bp_tree auto const new_insert_pos { insert_pos - mid }; bool const insertion_into_new_node{ insert_pos >= mid }; - decltype( auto ) key_to_propagate{ insertion_into_new_node + auto key_to_propagate{ insertion_into_new_node // we cannnot save a reference here because it might get invalidated by the new_node() call below ? base::insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) : base::insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) }; From 076b3c2624ce0f1f61be2e74449664b441fa8565 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 11:08:16 +0200 Subject: [PATCH 08/49] Minor readme update. --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5175693..1bb794f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,21 @@ -### vm - portable, lightweight, powerful, near-zero-overhead memory mapping and virtual memory management. +### vm - portable, lightweight, powerful, near-zero-overhead memory mapping, virtual memory management, containers and utilities. +##### Sponsored by (at one point or another) +https://farseer.com +https://microblink.com + +##### Some ancient discussions https://svn.boost.org/trac/boost/ticket/4827 http://boost.2283326.n4.nabble.com/interprocess-shared-memory-lifetime-td2603982.html http://lists.boost.org/Archives/boost/2010/10/172227.php http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n2044.html -##### Submodule requirements - * config_ex - * err - * std_fix +##### Dependencies + * Boost + * Psi.Build + * Psi.Err + * Psi.StdFix -##### C++ In-The-Kernel Now! ##### Copyright © 2011 - 2024. Domagoj Šarić. All rights reserved. From 316c3a0665e96c4400a90abbdc414844eb0a0c87 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 11:09:51 +0200 Subject: [PATCH 09/49] Defaulted vector::map_memory() initial_capacity argument to 0. --- include/psi/vm/vector.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index d85c4f4..8a5a7c2 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -1074,7 +1074,7 @@ class vector /////////////////////////////////////////////////////////////////////////// auto map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_file ( file, policy ); } - auto map_memory( size_type const size ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_memory( to_byte_sz( size ) ); } + auto map_memory( size_type const initial_capacity = 0 ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_memory( to_byte_sz( initial_capacity ) ); } bool has_attached_storage() const noexcept { return static_cast( storage_ ); } From 593039c50cf5bb0637aec9907a8d4b177f1cc9cf Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 11:12:46 +0200 Subject: [PATCH 10/49] Added a test for anonymous memory backed vectors. --- test/vector.cpp | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/test/vector.cpp b/test/vector.cpp index 49a4906..f3bacde 100644 --- a/test/vector.cpp +++ b/test/vector.cpp @@ -2,21 +2,30 @@ #include -#include #include -#include //------------------------------------------------------------------------------ namespace psi::vm { //------------------------------------------------------------------------------ -TEST( vector, playground ) +TEST( vector, anon_memory_backed ) { - std::filesystem::path const test_vec{ "test_vec" }; - std::filesystem::remove( test_vec ); + psi::vm::vector vec; + vec.map_memory(); + EXPECT_EQ( vec.size(), 0 ); + vec.append_range({ 3.14, 0.14, 0.04 }); + EXPECT_EQ( vec.size(), 3 ); + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); +} + +TEST( vector, file_backed ) +{ + auto const test_vec{ "test.vec" }; { - psi::vm::vector< double, std::uint16_t > vec; - vec.map_file( test_vec.c_str(), flags::named_object_construction_policy::create_new ); + psi::vm::vector vec; + vec.map_file( test_vec, flags::named_object_construction_policy::create_new_or_truncate_existing ); EXPECT_EQ( vec.size(), 0 ); vec.append_range({ 3.14, 0.14, 0.04 }); EXPECT_EQ( vec.size(), 3 ); @@ -25,8 +34,8 @@ TEST( vector, playground ) EXPECT_EQ( vec[ 2 ], 0.04 ); } { - psi::vm::vector< double, std::uint16_t > vec; - vec.map_file( test_vec.c_str(), flags::named_object_construction_policy::open_existing ); + psi::vm::vector vec; + vec.map_file( test_vec, flags::named_object_construction_policy::open_existing ); EXPECT_EQ( vec.size(), 3 ); EXPECT_EQ( vec[ 0 ], 3.14 ); EXPECT_EQ( vec[ 1 ], 0.14 ); From dc410d8948ed728ba7a69ec42ac20dca18f6bbcd Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 13:27:12 +0200 Subject: [PATCH 11/49] Windows: - 'fixed'/emulated support for resizeable pagefile backed mappings - minor shared mem cleanup/update - optimized the mem_info() function. --- README.md | 16 +++--- include/psi/vm/detail/nt.hpp | 51 ++++++++++++++++--- .../shared_memory/win32/mem.hpp | 14 +---- src/allocation/allocation.win32.cpp | 12 ++--- src/mappable_objects/file/file.win32.cpp | 9 ++++ src/mapped_view/mapped_view.win32.cpp | 2 +- src/mapping/mapping.win32.cpp | 43 +++++++++------- 7 files changed, 97 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 1bb794f..3505e75 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,20 @@ ### vm - portable, lightweight, powerful, near-zero-overhead memory mapping, virtual memory management, containers and utilities. +(TODO: sync with https://github.com/ned14/llfio) + ##### Sponsored by (at one point or another) -https://farseer.com -https://microblink.com +https://farseer.com +https://microblink.com + ##### Some ancient discussions -https://svn.boost.org/trac/boost/ticket/4827 -http://boost.2283326.n4.nabble.com/interprocess-shared-memory-lifetime-td2603982.html -http://lists.boost.org/Archives/boost/2010/10/172227.php -http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n2044.html + +https://svn.boost.org/trac/boost/ticket/4827 +http://boost.2283326.n4.nabble.com/interprocess-shared-memory-lifetime-td2603982.html +http://lists.boost.org/Archives/boost/2010/10/172227.php +http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2006/n2044.html ##### Dependencies * Boost diff --git a/include/psi/vm/detail/nt.hpp b/include/psi/vm/detail/nt.hpp index 59d0645..fc2e465 100644 --- a/include/psi/vm/detail/nt.hpp +++ b/include/psi/vm/detail/nt.hpp @@ -30,6 +30,8 @@ namespace psi::vm::nt { //------------------------------------------------------------------------------ +// TODO move to https://github.com/winsiderss/phnt + #ifndef STATUS_SUCCESS using NTSTATUS = LONG; NTSTATUS constexpr STATUS_SUCCESS{ 0 }; @@ -37,6 +39,9 @@ NTSTATUS constexpr STATUS_SUCCESS{ 0 }; #ifndef STATUS_CONFLICTING_ADDRESSES auto constexpr STATUS_CONFLICTING_ADDRESSES{ NTSTATUS( 0xC0000018 ) }; #endif +#ifndef STATUS_SECTION_NOT_EXTENDED +auto constexpr STATUS_SECTION_NOT_EXTENDED{ NTSTATUS( 0xC0000087 ) }; +#endif inline auto const current_process{ reinterpret_cast( std::intptr_t{ -1 } ) }; @@ -63,7 +68,7 @@ using NtCreateSection_t = NTSTATUS (NTAPI*) IN ULONG PageAttributess, IN ULONG SectionAttributes, IN HANDLE FileHandle OPTIONAL -); +) noexcept; inline auto const NtCreateSection{ detail::get_nt_proc( "NtCreateSection" ) }; enum SECTION_INFORMATION_CLASS { SectionBasicInformation, SectionImageInformation }; @@ -82,14 +87,14 @@ using NtQuerySection_t = NTSTATUS (NTAPI*) OUT PVOID InformationBuffer, IN ULONG InformationBufferSize, OUT PULONG ResultLength OPTIONAL -); +) noexcept; inline auto const NtQuerySection{ detail::get_nt_proc( "NtQuerySection" ) }; using NtExtendSection_t = NTSTATUS (NTAPI*) ( IN HANDLE SectionHandle, IN PLARGE_INTEGER NewSectionSize -); +) noexcept; inline auto const NtExtendSection{ detail::get_nt_proc( "NtExtendSection" ) }; enum SECTION_INHERIT @@ -109,12 +114,44 @@ using NtMapViewOfSection_t = NTSTATUS (NTAPI*) IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType OPTIONAL, IN ULONG Protect -); +) noexcept; inline auto const NtMapViewOfSection{ detail::get_nt_proc( "NtMapViewOfSection" ) }; -using NtAllocateVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, ULONG_PTR ZeroBits, PSIZE_T RegionSize, ULONG allocation_type, ULONG Protect ); -using NtFreeVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN PVOID * BaseAddress, PSIZE_T RegionSize, ULONG FreeType ); -using NtProtectVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, IN OUT PULONG NumberOfBytesToProtect, IN ULONG NewAccessProtection, OUT PULONG OldAccessProtection ); +enum MEMORY_INFORMATION_CLASS +{ + MemoryBasicInformation, // q: MEMORY_BASIC_INFORMATION + MemoryWorkingSetInformation, // q: MEMORY_WORKING_SET_INFORMATION + MemoryMappedFilenameInformation, // q: UNICODE_STRING + MemoryRegionInformation, // q: MEMORY_REGION_INFORMATION + MemoryWorkingSetExInformation, // q: MEMORY_WORKING_SET_EX_INFORMATION // since VISTA + MemorySharedCommitInformation, // q: MEMORY_SHARED_COMMIT_INFORMATION // since WIN8 + MemoryImageInformation, // q: MEMORY_IMAGE_INFORMATION + MemoryRegionInformationEx, // MEMORY_REGION_INFORMATION + MemoryPrivilegedBasicInformation, // MEMORY_BASIC_INFORMATION + MemoryEnclaveImageInformation, // MEMORY_ENCLAVE_IMAGE_INFORMATION // since REDSTONE3 + MemoryBasicInformationCapped, // 10 + MemoryPhysicalContiguityInformation, // MEMORY_PHYSICAL_CONTIGUITY_INFORMATION // since 20H1 + MemoryBadInformation, // since WIN11 + MemoryBadInformationAllProcesses, // since 22H1 + MemoryImageExtensionInformation, // MEMORY_IMAGE_EXTENSION_INFORMATION // since 24H2 + MaxMemoryInfoClass +}; + +using NtQueryVirtualMemory_t = NTSTATUS (NTAPI *) +( + HANDLE ProcessHandle, + PVOID BaseAddress, + MEMORY_INFORMATION_CLASS MemoryInformationClass, + PVOID MemoryInformation, + SIZE_T MemoryInformationLength, + PSIZE_T ReturnLength +) noexcept; +inline auto const NtQueryVirtualMemory{ detail::get_nt_proc( "NtQueryVirtualMemory" ) }; + + +using NtAllocateVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, ULONG_PTR ZeroBits, PSIZE_T RegionSize, ULONG allocation_type, ULONG Protect ) noexcept; +using NtFreeVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN PVOID * BaseAddress, PSIZE_T RegionSize, ULONG FreeType ) noexcept; +using NtProtectVirtualMemory_t = NTSTATUS (NTAPI*)( IN HANDLE ProcessHandle, IN OUT PVOID * BaseAddress, IN OUT PULONG NumberOfBytesToProtect, IN ULONG NewAccessProtection, OUT PULONG OldAccessProtection ) noexcept; inline auto const NtAllocateVirtualMemory{ detail::get_nt_proc( "NtAllocateVirtualMemory" ) }; inline auto const NtFreeVirtualMemory { detail::get_nt_proc( "NtFreeVirtualMemory" ) }; diff --git a/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp b/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp index 88838fb..5e6b8ed 100644 --- a/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp +++ b/include/psi/vm/mappable_objects/shared_memory/win32/mem.hpp @@ -339,8 +339,7 @@ class native_named_memory // { public: static - fallible_result - BOOST_CC_REG create + fallible_result create ( char const * const name, std::size_t const size, @@ -350,16 +349,7 @@ class native_named_memory // return detail::create_mapping_impl::do_map( file_handle::reference{ file_handle::traits::invalid_value }, flags, size, name ); } - fallible_result size() const noexcept - { - auto const p_view( ::MapViewOfFile( get(), 0, 0, 0, 0 ) ); - if ( BOOST_UNLIKELY( !p_view ) ) return error(); - MEMORY_BASIC_INFORMATION info; - BOOST_VERIFY( ::VirtualQuery( p_view, &info, sizeof( info ) ) == sizeof( info ) ); - BOOST_VERIFY( ::UnmapViewOfFile( p_view ) ); - BOOST_ASSUME( info.RegionSize % page_size == 0 ); - return info.RegionSize; - } + auto size() const noexcept { return get_size( *this ); } private: /// \note Required to enable the emplacement constructors of err diff --git a/src/allocation/allocation.win32.cpp b/src/allocation/allocation.win32.cpp index 7977c4f..de57382 100644 --- a/src/allocation/allocation.win32.cpp +++ b/src/allocation/allocation.win32.cpp @@ -93,11 +93,11 @@ void dealloc( void * & address, std::size_t & size, deallocation_type const type #endif } -MEMORY_BASIC_INFORMATION mem_info( void * const address ) noexcept +WIN32_MEMORY_REGION_INFORMATION mem_info( void * const address ) noexcept { - MEMORY_BASIC_INFORMATION info; - BOOST_VERIFY( ::VirtualQueryEx( nt::current_process, address, &info, sizeof( info ) ) == sizeof( info ) ); - BOOST_ASSUME( info.BaseAddress == address ); + WIN32_MEMORY_REGION_INFORMATION info; + BOOST_VERIFY( nt::NtQueryVirtualMemory( nt::current_process, address, nt::MemoryRegionInformation, &info, sizeof( info ), nullptr ) == nt::STATUS_SUCCESS ); + BOOST_ASSUME( info.AllocationBase == address ); return info; } @@ -119,9 +119,7 @@ bool commit( void * const desired_location, std::size_t const size ) noexcept { auto const info{ mem_info( final_address ) }; BOOST_ASSUME( info.AllocationProtect == PAGE_READWRITE ); - BOOST_ASSUME( info.State == MEM_RESERVE ); - BOOST_ASSUME( info.Protect == 0 ); - BOOST_ASSUME( info.Type == MEM_PRIVATE ); + BOOST_ASSUME( info.Private ); auto region_size{ std::min( static_cast< std::size_t >( info.RegionSize ), size - final_size ) }; auto const partial_result{ alloc( final_address, region_size, allocation_type::commit ) }; if ( partial_result != STATUS_SUCCESS ) diff --git a/src/mappable_objects/file/file.win32.cpp b/src/mappable_objects/file/file.win32.cpp index a59753e..133ea1c 100644 --- a/src/mappable_objects/file/file.win32.cpp +++ b/src/mappable_objects/file/file.win32.cpp @@ -118,6 +118,15 @@ namespace detail HANDLE handle{ handle_traits::invalid_value }; LARGE_INTEGER maximum_size{ .QuadPart = static_cast( size ) }; BOOST_ASSERT_MSG( std::uint64_t( maximum_size.QuadPart ) == size, "Unsupported section size" ); + if ( !file ) + { + // Windows (11 23H2) does not (still) seem to support resizing of pagefile-backed mappings + // so we emulate those by creating a 4GB one and counting on: + // - the NT kernel to be lazy and + // - that amount to be 'enough for everyone'. + maximum_size.QuadPart = std::numeric_limits::max(); + } + auto const nt_result { nt::NtCreateSection // TODO use it for named sections also diff --git a/src/mapped_view/mapped_view.win32.cpp b/src/mapped_view/mapped_view.win32.cpp index 9793a57..1c83636 100644 --- a/src/mapped_view/mapped_view.win32.cpp +++ b/src/mapped_view/mapped_view.win32.cpp @@ -100,7 +100,7 @@ map } // defined in allocation.win32.cpp -MEMORY_BASIC_INFORMATION mem_info( void * const ) noexcept; +WIN32_MEMORY_REGION_INFORMATION mem_info( void * const ) noexcept; namespace { diff --git a/src/mapping/mapping.win32.cpp b/src/mapping/mapping.win32.cpp index d4efb54..c3bc16c 100644 --- a/src/mapping/mapping.win32.cpp +++ b/src/mapping/mapping.win32.cpp @@ -34,7 +34,7 @@ std::uint64_t get_size( mapping::const_handle const mapping_handle ) noexcept using namespace nt; SECTION_BASIC_INFORMATION info; auto const result{ NtQuerySection( mapping_handle.value, SECTION_INFORMATION_CLASS::SectionBasicInformation, &info, sizeof( info ), nullptr ) }; - BOOST_VERIFY( NT_SUCCESS( result ) ); + BOOST_ASSUME( result == STATUS_SUCCESS ); return info.SectionSize.QuadPart; } @@ -43,24 +43,33 @@ namespace detail::create_mapping_impl { HANDLE map_file( file_handle::reference err::fallible_result set_size( mapping & the_mapping, std::uint64_t const new_size ) noexcept { using namespace nt; - LARGE_INTEGER ntsz{ .QuadPart = static_cast( new_size ) }; - auto const result{ NtExtendSection( the_mapping.get(), &ntsz ) }; - if ( !NT_SUCCESS( result ) ) [[ unlikely ]] - return result; - BOOST_ASSERT( ntsz.QuadPart >= static_cast( new_size ) ); - if ( ntsz.QuadPart > static_cast( new_size ) ) + if ( the_mapping.is_file_based() ) { - // NtExtendSection does not support downsizing - use it also as a size getter (avoid get_size call) - BOOST_ASSERT( ntsz.QuadPart == static_cast( get_size( the_mapping ) ) ); - the_mapping.close(); // no strong guarantee :/ - auto const file_reisze_result{ set_size( the_mapping.underlying_file(), new_size )() }; - if ( !file_reisze_result ) - return nt::error/*...mrmlj...*/( file_reisze_result.error().get() ); // TODO fully move to NativeNT API https://cpp.hotexamples.com/examples/-/-/NtSetInformationFile/cpp-ntsetinformationfile-function-examples.html - auto const new_mapping_handle{ detail::create_mapping_impl::map_file( the_mapping.file, the_mapping.create_mapping_flags, new_size ) }; - the_mapping.reset( new_mapping_handle ); - if ( !the_mapping ) - return nt::error/*...mrmlj...*/( err::last_win32_error::get() ); + LARGE_INTEGER ntsz{ .QuadPart = static_cast( new_size ) }; + auto const result{ NtExtendSection( the_mapping.get(), &ntsz ) }; + if ( !NT_SUCCESS( result ) ) [[ unlikely ]] + return result; + + BOOST_ASSERT( ntsz.QuadPart >= static_cast( new_size ) ); + if ( ntsz.QuadPart > static_cast( new_size ) ) + { + // NtExtendSection does not support downsizing - use it also as a size getter (avoid get_size call) + BOOST_ASSERT( ntsz.QuadPart == static_cast( get_size( the_mapping ) ) ); + the_mapping.close(); // no strong guarantee :/ + auto const file_reisze_result{ set_size( the_mapping.underlying_file(), new_size )() }; + if ( !file_reisze_result ) + return nt::error/*...mrmlj...*/( file_reisze_result.error().get() ); // TODO fully move to NativeNT API https://cpp.hotexamples.com/examples/-/-/NtSetInformationFile/cpp-ntsetinformationfile-function-examples.html + auto const new_mapping_handle{ detail::create_mapping_impl::map_file( the_mapping.file, the_mapping.create_mapping_flags, new_size ) }; + the_mapping.reset( new_mapping_handle ); + if ( !the_mapping ) + return nt::error/*...mrmlj...*/( err::last_win32_error::get() ); + } + } + else + { + if ( new_size > get_size( the_mapping ) ) [[ unlikely ]] + return nt::STATUS_SECTION_NOT_EXTENDED; } return err::success; } From 54170222f1619f5eb9bf0e54d4d22eb1b3b40832 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 23:18:27 +0200 Subject: [PATCH 12/49] Linux: quick-wrkrnd for an old yet still alive kernel bug. --- include/psi/vm/vector.hpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index 8a5a7c2..a199baa 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -155,6 +155,11 @@ class contiguous_container_storage_base std::move( file ), ap::object{ ap::readwrite }, ap::child_process::does_not_inherit, +# ifdef __linux__ + // TODO solve in a cleaner/'in a single place' way + // https://bugzilla.kernel.org/show_bug.cgi?id=8691 mremap: Wrong behaviour expanding a MAP_SHARED anonymous mapping + !file ? flags::share_mode::hidden : +# endif flags::share_mode::shared, mapping::supports_zero_sized_mappings ? mapping_size @@ -176,7 +181,7 @@ class contiguous_container_storage_base mapping mapping_; }; // contiguous_container_storage_base -template < typename sz_t, bool headerless > +template class contiguous_container_storage : public contiguous_container_storage_base, @@ -856,10 +861,11 @@ class vector reference emplace_back( Args &&...args ) { storage_.expand( to_byte_sz( size() + 1 ) ); + auto & placeholder{ back() }; if constexpr ( sizeof...( args ) ) - return *std::construct_at( &back(), std::forward< Args >( args )... ); + return *std::construct_at( &placeholder, std::forward( args )... ); else - return *new ( &back() ) T; // default init + return *new ( &placeholder ) T; // default init } //! Effects: Inserts an object of type T constructed with From 4661eafe22cd7e222386cdd7cae48e1e4570ac50 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 23:48:58 +0200 Subject: [PATCH 13/49] Fixed logic and edge-case handling around node min-mid-max value/children counts (and how it relates to node splitting and merging). --- include/psi/vm/containers/b+tree.hpp | 156 ++++++++++++++++----------- 1 file changed, 95 insertions(+), 61 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 283043b..b0e2ee5 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -77,6 +77,10 @@ class bptree_base using depth_t = std::uint8_t; + template + // ceil( m / 2 ) + static constexpr auto ihalf_ceil{ static_cast( ( value + 1 ) / 2 ) }; + struct [[ nodiscard, clang::trivial_abi ]] node_slot // instead of node pointers we store offsets - slots in the node pool { using value_type = std::uint32_t; @@ -194,7 +198,9 @@ class bptree_base ( N const & source, node_size_type const srcBegin, node_size_type const srcEnd, N & target, node_size_type const tgtBegin - ) noexcept { + ) noexcept + { + BOOST_ASSUME( &source != &target ); // otherwise could require move_backwards or shift_* BOOST_ASSUME( srcBegin <= srcEnd ); std::uninitialized_move( &(source.*array)[ srcBegin ], &(source.*array)[ srcEnd ], &(target.*array)[ tgtBegin ] ); } @@ -309,8 +315,8 @@ class bptree_base_wkey : public bptree_base using value_type = Key; - static auto constexpr max_children{ order }; - static auto constexpr max_values { max_children - 1 }; + static node_size_type constexpr max_children{ order }; + static node_size_type constexpr max_values { max_children - 1 }; node_slot children[ max_children ]; Key keys [ max_values ]; @@ -318,8 +324,8 @@ class bptree_base_wkey : public bptree_base struct inner_node : parent_node { - static auto constexpr min_children{ ( parent_node::max_children + 1 ) / 2 }; // ceil( m / 2 ) - static auto constexpr min_values { min_children - 1 }; + static node_size_type constexpr min_children{ ihalf_ceil }; + static node_size_type constexpr min_values { min_children - 1 }; static_assert( min_children >= 3 ); }; // struct inner_node @@ -335,9 +341,9 @@ class bptree_base_wkey : public bptree_base // TODO support for maps (i.e. keys+values) using value_type = Key; - static auto constexpr storage_space{ node_size - psi::vm::align_up( sizeof( linked_node_header ), alignof( Key ) ) }; - static auto constexpr max_values { storage_space / sizeof( Key ) }; - static auto constexpr min_values { ( max_values + 1 ) / 2 }; + static node_size_type constexpr storage_space{ node_size - psi::vm::align_up( sizeof( linked_node_header ), alignof( Key ) ) }; + static node_size_type constexpr max_values { storage_space / sizeof( Key ) }; + static node_size_type constexpr min_values { ihalf_ceil }; Key keys[ max_values ]; }; // struct leaf_node @@ -362,21 +368,22 @@ class bptree_base_wkey : public bptree_base protected: // helpers for split_to_insert static value_type insert_into_new_node ( - parent_node & node, parent_node & new_node, + inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_size_type const new_insert_pos, node_slot const key_right_child ) noexcept { BOOST_ASSUME( bool( key_right_child ) ); - using N = parent_node; + using N = inner_node; - auto const max{ N::max_values }; // or 'order' - auto const mid{ max / 2 }; + auto const max{ N::max_values }; + auto const mid{ N::min_values }; BOOST_ASSUME( node.num_vals == max ); BOOST_ASSUME( new_node.num_vals == 0 ); + // old node gets the minimum/median/mid -> the new gets the leftovers (i.e. mid or mid + 1) new_node.num_vals = max - mid; value_type key_to_propagate; @@ -403,6 +410,9 @@ class bptree_base_wkey : public bptree_base node.num_vals = mid; + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + return key_to_propagate; } @@ -417,8 +427,8 @@ class bptree_base_wkey : public bptree_base using N = leaf_node; - auto const max{ N::max_values }; // or 'order' - auto const mid{ max / 2 }; + auto const max{ N::max_values }; + auto const mid{ N::min_values }; BOOST_ASSUME( node.num_vals == max ); BOOST_ASSUME( new_node.num_vals == 0 ); @@ -431,17 +441,21 @@ class bptree_base_wkey : public bptree_base keys( new_node )[ new_insert_pos ] = value; auto const & key_to_propagate{ new_node.keys[ 0 ] }; + + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + return key_to_propagate; } - static value_type insert_into_existing_node( parent_node & node, parent_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + static value_type insert_into_existing_node( inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept { BOOST_ASSUME( bool( key_right_child ) ); - using N = parent_node; + using N = inner_node; - auto const max{ N::max_values }; // or 'order' - auto const mid{ max / 2 }; + auto const max{ N::max_values }; + auto const mid{ N::min_values }; BOOST_ASSUME( node.num_vals == max ); BOOST_ASSUME( new_node.num_vals == 0 ); @@ -460,6 +474,9 @@ class bptree_base_wkey : public bptree_base keys ( node )[ insert_pos ] = value; children( node )[ insert_pos + 1 ] = key_right_child; + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + return key_to_propagate; } @@ -469,20 +486,24 @@ class bptree_base_wkey : public bptree_base using N = leaf_node; - auto const max{ N::max_values }; // or 'order' - auto const mid{ max / 2 }; + auto const max{ N::max_values }; + auto const mid{ N::min_values }; BOOST_ASSUME( node.num_vals == max ); BOOST_ASSUME( new_node.num_vals == 0 ); - umove<&N::keys> ( node, mid, max, new_node, 0 ); - std::shift_right( &node.keys[ insert_pos ], &node.keys[ mid + 1 ], 1 ); + umove<&N::keys> ( node, mid - 1, max, new_node, 0 ); + std::shift_right( &node.keys[ insert_pos ], &node.keys[ mid ], 1 ); - node .num_vals = mid + 1 ; - new_node.num_vals = max - mid; + node .num_vals = mid; + new_node.num_vals = max - mid + 1; keys( node )[ insert_pos ] = value; auto const & key_to_propagate{ new_node.keys[ 0 ] }; + + BOOST_ASSUME( !underflowed( node ) ); + BOOST_ASSUME( !underflowed( new_node ) ); + return key_to_propagate; } @@ -505,15 +526,17 @@ class bptree_base_wkey : public bptree_base inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx ) noexcept { - BOOST_ASSUME( source.num_vals <= source.max_values ); BOOST_ASSUME( source.num_vals >= source.min_values - 1 ); - BOOST_ASSUME( target.num_vals <= target.max_values ); BOOST_ASSUME( target.num_vals >= target.min_values - 1 ); + auto constexpr min{ leaf_node::min_values }; + BOOST_ASSUME( source.num_vals >= min - 1 ); BOOST_ASSUME( source.num_vals <= min ); + BOOST_ASSUME( target.num_vals >= min - 1 ); BOOST_ASSUME( target.num_vals <= min ); std::ranges::move( keys ( source ), keys ( target ).end() ); std::ranges::move( children( source ), children( target ).end() ); target.num_vals += source.num_vals; source.num_vals = 0; target.next = source.next; - BOOST_ASSUME( target.num_vals == target.max_values - 1 ); + BOOST_ASSUME( target.num_vals >= 2 * min - 2 ); + BOOST_ASSUME( target.num_vals <= 2 * min - 1 ); std::ranges::shift_left( keys ( parent ).subspan( parent_key_idx ), 1 ); std::ranges::shift_left( children( parent ).subspan( parent_child_idx ), 1 ); BOOST_ASSUME( parent.num_vals ); @@ -531,8 +554,9 @@ class bptree_base_wkey : public bptree_base inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx ) noexcept { - BOOST_ASSUME( right.num_vals <= right.max_values ); BOOST_ASSUME( right.num_vals >= right.min_values - 1 ); - BOOST_ASSUME( left .num_vals <= left .max_values ); BOOST_ASSUME( left .num_vals >= left .min_values - 1 ); + auto constexpr min{ inner_node::min_values }; + BOOST_ASSUME( right.num_vals >= min - 1 ); BOOST_ASSUME( right.num_vals <= min ); + BOOST_ASSUME( left .num_vals >= min - 1 ); BOOST_ASSUME( left .num_vals <= min ); for ( auto const ch_slot : children( right ) ) this->node( ch_slot ).parent = slot_of( left ); @@ -643,6 +667,7 @@ class bp_tree using bptree_base::as; using bptree_base::can_borrow; using bptree_base::children; + using bptree_base::ihalf_ceil; using bptree_base::keys; using bptree_base::node; using bptree_base::num_chldrn; @@ -727,7 +752,10 @@ class bp_tree leaf_node & leaf{ nodes.leaf }; if ( depth_ != 1 ) + { verify( leaf ); + BOOST_ASSUME( leaf.num_vals >= leaf.min_values ); + } auto const key_offset{ find( leaf, key ) }; if ( nodes.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" @@ -752,7 +780,8 @@ class bp_tree BOOST_ASSUME( root_ == slot_of( leaf ) ); BOOST_ASSUME( leaf.is_root() ); BOOST_ASSUME( size_ == leaf.num_vals + 1 ); - if ( leaf.num_vals == 0 ) { + if ( leaf.num_vals == 0 ) + { root_ = {}; base::free( leaf ); --depth_; @@ -792,8 +821,8 @@ class bp_tree void split_to_insert( N * p_node, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) { verify( *p_node ); - node_size_type const max{ N::max_values }; // or 'order' - node_size_type const mid{ max / 2 }; + auto const max{ N::max_values }; + auto const mid{ N::min_values }; auto const node_slot{ slot_of( *p_node ) }; auto * p_new_node{ &bptree_base::new_node() }; @@ -812,8 +841,8 @@ class bp_tree p_node ->next = new_slot; } - auto const new_insert_pos { insert_pos - mid }; - bool const insertion_into_new_node{ insert_pos >= mid }; + auto const new_insert_pos { insert_pos - mid }; + bool const insertion_into_new_node{ new_insert_pos >= 0 }; auto key_to_propagate{ insertion_into_new_node // we cannnot save a reference here because it might get invalidated by the new_node() call below ? base::insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) : base::insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) @@ -821,6 +850,7 @@ class bp_tree verify( *p_node ); verify( *p_new_node ); + BOOST_ASSUME( p_node->num_vals == mid ); if constexpr ( parent_node_type ) { for ( auto const ch_slot : children( *p_new_node ) ) { @@ -848,18 +878,19 @@ class bp_tree } template - void insert( N & target_node, key_const_arg v, node_slot const rightChild ) { + void insert( N & target_node, key_const_arg v, node_slot const right_child ) + { verify( target_node ); auto const pos{ find( target_node, v ) }; if ( base::full( target_node ) ) [[ unlikely ]] { - return split_to_insert( &target_node, pos, v, rightChild ); + return split_to_insert( &target_node, pos, v, right_child ); } else { ++target_node.num_vals; std::ranges::shift_right( base::keys( target_node ).subspan( pos ), 1 ); target_node.keys[ pos ] = v; if constexpr ( requires { target_node.children; } ) { std::ranges::shift_right( children( target_node ).subspan( pos + /*>right< child*/1 ), 1 ); - target_node.children[ pos + 1 ] = rightChild; + target_node.children[ pos + 1 ] = right_child; } } } @@ -867,7 +898,8 @@ class bp_tree template BOOST_NOINLINE - void handle_underflow( N & node, depth_t const level ) { + void handle_underflow( N & node, depth_t const level ) + { auto & __restrict depth_{ this->hdr().depth_ }; auto & __restrict root_ { this->hdr().root_ }; @@ -876,21 +908,6 @@ class bp_tree auto constexpr parent_node_type{ requires{ node.children; } }; auto constexpr leaf_node_type { !parent_node_type }; - if ( node.is_root() ) [[ unlikely ]] { - BOOST_ASSUME( level == 0 ); - BOOST_ASSUME( level < depth_ ); // 'leaf root' should be handled directly by the special case in erase() - BOOST_ASSUME( root_ == slot_of( node ) ); - if constexpr ( parent_node_type ) { - BOOST_ASSUME( !!node.children[ 0 ] ); - root_ = node.children[ 0 ]; - BOOST_ASSUME( depth_ > 1 ); - } - bptree_base::node( root_ ).parent = {}; - free( node ); - --depth_; - return; - } - BOOST_ASSUME( level > 0 ); if constexpr ( leaf_node_type ) { BOOST_ASSUME( level == leaf_level() ); @@ -899,6 +916,8 @@ class bp_tree auto & parent{ this->node( node.parent ) }; verify( parent ); + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); + auto const parent_key_idx { find( parent, node.keys[ 0 ] ) }; BOOST_ASSUME( parent_key_idx != parent.num_vals || !leaf_node_type ); bool const parent_has_key_copy{ leaf_node_type && ( parent.keys[ parent_key_idx ] == node.keys[ 0 ] ) }; auto const parent_child_idx { static_cast( parent_key_idx + parent_has_key_copy ) }; @@ -921,9 +940,9 @@ class bp_tree // Borrow from left sibling if possible if ( p_left_sibling && can_borrow( *p_left_sibling ) ) { verify( *p_left_sibling ); - + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); node.num_vals++; - BOOST_ASSUME( node.num_vals <= node.max_values ); + BOOST_ASSUME( node.num_vals == node.min_values ); auto const node_vals{ keys( node ) }; auto const left_vals{ keys( *p_left_sibling ) }; @@ -960,7 +979,7 @@ class bp_tree verify( *p_right_sibling ); node.num_vals++; - BOOST_ASSUME( node.num_vals <= node.max_values ); + BOOST_ASSUME( node.num_vals == node.min_values ); if constexpr ( leaf_node_type ) { // Move the smallest key from right sibling to current node the parent @@ -991,10 +1010,12 @@ class bp_tree // Merge with left or right sibling else { if ( p_left_sibling ) { + verify( *p_left_sibling ); BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); // Merge node -> left sibling this->merge_nodes( node, *p_left_sibling, parent, left_separator_key_idx, parent_child_idx ); } else { + verify( *p_right_sibling ); BOOST_ASSUME( parent_key_idx == 0 ); BOOST_ASSUME( parent.keys[ parent_key_idx ] <= p_right_sibling->keys[ 0 ] ); // Merge right sibling -> node @@ -1002,17 +1023,30 @@ class bp_tree } // propagate underflow - if ( - ( ( level == 1 ) && underflowed( as( parent ) ) ) || - ( ( level != 1 ) && underflowed( parent ) ) - ) + if ( parent.is_root() ) [[ unlikely ]] + { + BOOST_ASSUME( root_ == slot_of( parent ) ); + BOOST_ASSUME( level == 1 ); + BOOST_ASSUME( depth_ > level ); // 'leaf root' should be handled directly by the special case in erase() + auto & root{ as( parent ) }; + BOOST_ASSUME( !!root.children[ 0 ] ); + if ( underflowed( root ) ) + { + root_ = root.children[ 0 ]; + bptree_base::node( root_ ).parent = {}; + --depth_; + free( root ); + } + } + else + if ( underflowed( parent ) ) { BOOST_ASSUME( level > 0 ); BOOST_ASSUME( level < depth_ ); handle_underflow( parent, level - 1 ); } } - } + } // handle_underflow() struct key_locations From 518a9ec871a6304080668f1c56975d0a904ec70b Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 28 Sep 2024 23:49:17 +0200 Subject: [PATCH 14/49] Minor cleanup. --- src/mappable_objects/file/file.posix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mappable_objects/file/file.posix.cpp b/src/mappable_objects/file/file.posix.cpp index d0e2ef5..a04c4ce 100644 --- a/src/mappable_objects/file/file.posix.cpp +++ b/src/mappable_objects/file/file.posix.cpp @@ -110,7 +110,7 @@ mapping create_mapping # if defined( MAP_SHARED_VALIDATE ) // mmap fails with EINVAL (under WSL kernel 5.15 w/ ArchLinux) when // MAP_SHARED_VALIDATE is combined with MAP_ANONYMOUS - if ( ( std::to_underlying( flags::mapping::share_mode::shared ) == MAP_SHARED_VALIDATE ) && ( view_flags.flags & MAP_SHARED_VALIDATE ) ) + if ( ( view_flags.flags & MAP_TYPE ) == MAP_SHARED_VALIDATE ) { view_flags.flags &= ~MAP_SHARED_VALIDATE; view_flags.flags |= MAP_SHARED; From a7484d6a014a43f3805e04606fb24b0bb0429b38 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sun, 29 Sep 2024 00:51:43 +0200 Subject: [PATCH 15/49] Minor cleanups. --- include/psi/vm/flags/mapping.hpp | 1 - include/psi/vm/vector.hpp | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/include/psi/vm/flags/mapping.hpp b/include/psi/vm/flags/mapping.hpp index 92fd4ff..cef35bd 100644 --- a/include/psi/vm/flags/mapping.hpp +++ b/include/psi/vm/flags/mapping.hpp @@ -36,7 +36,6 @@ namespace flags /// //////////////////////////////////////////////////////////////////////////////// - struct mapping { enum struct share_mode diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index a199baa..e11e11e 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -395,8 +395,8 @@ struct header_info // to trivial_abi types. // Used the Boost.Container vector as a starting skeleton. -template < typename T, typename sz_t = std::size_t, bool headerless_param = true > -requires is_trivially_moveable< T > +template +requires is_trivially_moveable class vector { private: @@ -1099,7 +1099,7 @@ class vector //! Note: Non-standard extension. bool stable_reserve( size_type new_cap ) = /*TODO*/ delete; - void grow_to( size_type const target_size, default_init_t ) requires( std::is_trivial_v ) + void grow_to( size_type const target_size, default_init_t ) requires( std::is_trivially_default_constructible_v ) { storage_.expand( to_byte_sz( target_size ) ); } @@ -1126,10 +1126,7 @@ class vector } else { - for ( auto & element : span().subspan( current_size ) ) - { - std::construct_at( &element ); - } + std::uninitialized_default_construct( nth( current_size ), end() ); } } PSI_WARNING_DISABLE_POP() From 7fbd6fedf34ecb25e7c4ff0cff5e111f1b1e8fcf Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sun, 29 Sep 2024 00:54:08 +0200 Subject: [PATCH 16/49] Added the bptree*::reserve() member function (with not quite standard semantics). Fixed a bug in bptree_base::free(). Minor other stylistic changes. --- include/psi/vm/containers/b+tree.hpp | 22 ++++++++++++++++------ src/containers/b+tree.cpp | 28 ++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index b0e2ee5..8f8590a 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -147,6 +147,8 @@ class bptree_base void free( node_header & ) noexcept; + void reserve( node_slot::value_type additional_nodes ); + [[ gnu::pure ]] header & hdr() noexcept; [[ gnu::pure ]] header const & hdr() const noexcept { return const_cast( *this ).hdr(); } @@ -680,6 +682,7 @@ class bp_tree using base::root; public: + using size_type = base::size_type; using value_type = base::value_type; using pointer = value_type *; using const_pointer = value_type const *; @@ -712,8 +715,6 @@ class bp_tree auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end() }; } auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end() }; } - void swap( bp_tree & other ) noexcept { base::swap( other ); } - BOOST_NOINLINE void insert( key_const_arg v ) { @@ -798,6 +799,10 @@ class bp_tree --size_; } + void reserve( size_type const additional_values ) { bptree_base::reserve( ( additional_values + leaf_node::max_values - 1 ) / leaf_node::max_values ); } + + void swap( bp_tree & other ) noexcept { base::swap( other ); } + Comparator const & comp() const noexcept { return *this; } // UB if the comparator is changed in such a way as to invalidate to order of elements already in the container @@ -818,7 +823,8 @@ class bp_tree } template - void split_to_insert( N * p_node, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) { + void split_to_insert( N * p_node, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) + { verify( *p_node ); auto const max{ N::max_values }; @@ -938,7 +944,8 @@ class bp_tree BOOST_ASSERT( &node != p_right_sibling ); BOOST_ASSERT( static_cast( &node ) != static_cast( &parent ) ); // Borrow from left sibling if possible - if ( p_left_sibling && can_borrow( *p_left_sibling ) ) { + if ( p_left_sibling && can_borrow( *p_left_sibling ) ) + { verify( *p_left_sibling ); BOOST_ASSUME( node.num_vals == node.min_values - 1 ); node.num_vals++; @@ -975,7 +982,9 @@ class bp_tree BOOST_ASSUME( p_left_sibling->num_vals >= N::min_values ); } // Borrow from right sibling if possible - else if ( p_right_sibling && can_borrow( *p_right_sibling ) ) { + else + if ( p_right_sibling && can_borrow( *p_right_sibling ) ) + { verify( *p_right_sibling ); node.num_vals++; @@ -1008,7 +1017,8 @@ class bp_tree BOOST_ASSUME( p_right_sibling->num_vals >= N::min_values ); } // Merge with left or right sibling - else { + else + { if ( p_left_sibling ) { verify( *p_left_sibling ); BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index 076231f..d2e8e48 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -30,17 +30,28 @@ void bptree_base::clear() noexcept hdr() = {}; } -std::span bptree_base::user_header_data() noexcept { return header_data().second; } +std::span +bptree_base::user_header_data() noexcept { return header_data().second; } -bptree_base::header & bptree_base::hdr() noexcept { return *header_data().first; } +bptree_base::header & +bptree_base::hdr() noexcept { return *header_data().first; } -bptree_base::storage_result bptree_base::init_header( storage_result success ) noexcept +bptree_base::storage_result +bptree_base::init_header( storage_result success ) noexcept { if ( std::move( success ) && nodes_.empty() ) hdr() = {}; return success; } +void bptree_base::reserve( node_slot::value_type const additional_nodes ) +{ + auto const current_size{ nodes_.size() }; + nodes_.grow_by( additional_nodes, value_init ); + for ( auto & n : std::views::reverse( std::span{ nodes_ }.subspan( current_size ) ) ) + free( n ); +} + bptree_base::base_iterator::base_iterator( node_pool & nodes, node_slot const node_offset, node_size_type const value_offset ) noexcept : #ifndef NDEBUG // for bounds checking @@ -140,10 +151,11 @@ bptree_base::depth_t bptree_base:: leaf_level() const noexcept { BOOST_ASSUME( void bptree_base::free( node_header & node ) noexcept { + BOOST_ASSUME( node.num_vals == 0 ); auto & free_list{ hdr().free_list_ }; auto & free_node{ static_cast( node ) }; - if ( free_list ) - free_node.next = free_list; + if ( free_list ) free_node.next = free_list; + else free_node.next = {}; free_list = slot_of( free_node ); } @@ -161,10 +173,14 @@ bptree_base::new_node() if ( free_list ) { auto & cached_node{ node( free_list ) }; + BOOST_ASSUME( cached_node.num_vals == 0 ); free_list = cached_node.next; + cached_node.next = {}; return as( cached_node ); } - return nodes_.emplace_back(); + auto & new_node{ nodes_.emplace_back() }; + BOOST_ASSUME( new_node.num_vals == 0 ); + return new_node; } //------------------------------------------------------------------------------ From cb747887d46aedb5e538aef282ec855da33c2415 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Wed, 2 Oct 2024 10:46:37 +0200 Subject: [PATCH 17/49] Changed the node design so that - all node types at all levels form horizontal/breadth doubly linked lists - precise in-parent-position for child nodes is tracked (simplifying other code and eliminating the need for binary searches in a couple of places). Added testing for pure in-memory operation. Various related refactoring, cleanups ando optimizations. --- include/psi/vm/containers/b+tree.hpp | 798 ++++++++++++--------- include/psi/vm/containers/b+tree_print.hpp | 95 +-- src/containers/b+tree.cpp | 169 ++++- test/b+tree.cpp | 23 + 4 files changed, 664 insertions(+), 421 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 8f8590a..076439d 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -21,14 +21,6 @@ #include #include //------------------------------------------------------------------------------ -#if __cpp_lib_shift < 202202L -namespace std::ranges -{ - auto shift_left ( auto && rng, auto d ) noexcept { return std::shift_left ( rng.begin(), rng.end(), d ); } - auto shift_right( auto && rng, auto d ) noexcept { return std::shift_right( rng.begin(), rng.end(), d ); } -} -#endif -//------------------------------------------------------------------------------ namespace psi::vm { //------------------------------------------------------------------------------ @@ -95,11 +87,21 @@ class bptree_base { using size_type = std::uint16_t; - node_slot parent {}; - size_type num_vals{}; + // At minimum we need single-linked/directed list in the vertical/depth and horizontal/breadth directions + // (and the latter only for the leaf level - to have a connected sorted 'list' of all the values). + // However having a precise vertical back/up-link (parent_child_idx): + // * speeds up walks up the tree (as the parent (separator) key slots do not have to be searched for) + // * simplifies code (enabling several functions to become independent of the comparator - no longer + // need searching - and moved up into the base bptree classes) + // * while at the same time being a negligible overhead considering we are targeting much larger (page + // size sized) nodes. + node_slot parent {}; + node_slot left {}; + node_slot right {}; + size_type num_vals {}; + size_type parent_child_idx{}; /* TODO - size_type start; // make keys and children arrays function as devectors: allow empty space at the beginning to avoid moves for small borrowings - size_type in_parent_pos; // position of the node in its parent (to avoid finds in upscans). + size_type start; // make keys and children arrays function as devectors: allow empty space at the beginning to avoid moves for small borrowings */ [[ gnu::pure ]] bool is_root() const noexcept { return !parent; } @@ -112,10 +114,8 @@ class bptree_base }; // struct node_header using node_size_type = node_header::size_type; - struct linked_node_header : node_header { node_slot next{}; /*singly linked free list*/ }; - - struct alignas( node_size ) node_placeholder : node_header {}; - struct alignas( node_size ) free_node : linked_node_header {}; + struct alignas( node_size ) node_placeholder : node_header {}; + struct alignas( node_size ) free_node : node_header {}; // SCARY iterator parts class base_iterator; @@ -137,7 +137,7 @@ class bptree_base [[ gnu::pure ]] size_type size() const noexcept { return hdr().size_; } - [[ gnu::cold ]] linked_node_header & create_root(); + [[ gnu::cold ]] node_header & create_root(); [[ gnu::pure ]] static bool underflowed( auto const & node ) noexcept { return node.num_vals < node.min_values; } [[ gnu::pure ]] static bool can_borrow ( auto const & node ) noexcept { return node.num_vals > node.min_values; } @@ -170,6 +170,69 @@ class bptree_base [[ gnu::pure ]] static constexpr node_size_type num_vals ( auto const & node ) noexcept { return node.num_vals; } [[ gnu::pure ]] static constexpr node_size_type num_chldrn( auto const & node ) noexcept { if constexpr ( requires{ node.children; } ) { BOOST_ASSUME( node.num_vals ); return node.num_vals + 1U; } else return 0; } + template + static constexpr node_size_type size( auto const & node ) noexcept + { + if constexpr ( requires{ &(node.*array) == &node.keys; } ) return num_vals ( node ); + else return num_chldrn( node ); + } + + template + static auto rshift( auto & node, node_size_type const start_offset, node_size_type const end_offset ) noexcept + { + auto const max{ std::size( node.*array ) }; + BOOST_ASSUME( end_offset <= max ); + BOOST_ASSUME( start_offset < max ); + BOOST_ASSUME( start_offset < end_offset ); + auto const begin{ &(node.*array)[ start_offset ] }; + auto const end { &(node.*array)[ end_offset ] }; + auto const new_begin{ std::shift_right( begin, end, 1 ) }; + BOOST_ASSUME( new_begin == begin + 1 ); + return std::span{ new_begin, end }; + } + template static auto rshift( auto & node, node_size_type const offset ) noexcept { return rshift( node, offset, size( node ) ); } + template static auto rshift( auto & node ) noexcept { return rshift( node, 0 ); } + template + static auto lshift( auto & node, node_size_type const start_offset, node_size_type const end_offset ) noexcept + { + auto const max{ std::size( node.*array ) }; + BOOST_ASSUME( end_offset <= max ); + BOOST_ASSUME( start_offset < max ); + BOOST_ASSUME( start_offset < end_offset ); + auto const begin{ &(node.*array)[ start_offset ] }; + auto const end { &(node.*array)[ end_offset ] }; + auto const new_end{ std::shift_left( begin, end, 1 ) }; + BOOST_ASSUME( new_end == end - 1 ); + return std::span{ begin, new_end }; + } + template static auto lshift( auto & node, node_size_type const offset ) noexcept { return lshift( node, offset, size( node ) ); } + template static auto lshift( auto & node ) noexcept { return lshift( node, 0 ); } + + template static void rshift_keys( N & node, auto... args ) noexcept { rshift<&N::keys>( node, args... ); } + template static void lshift_keys( N & node, auto... args ) noexcept { lshift<&N::keys>( node, args... ); } + + template + void rshift_chldrn( N & parent, auto... args ) noexcept { + auto const shifted_children{ rshift<&N::children>( parent, args... ) }; + for ( auto ch_slot : shifted_children ) + node( ch_slot ).parent_child_idx++; + } + template + void lshift_chldrn( N & parent, auto... args ) noexcept { + auto const shifted_children{ lshift<&N::children>( parent, args... ) }; + for ( auto ch_slot : shifted_children ) + node( ch_slot ).parent_child_idx--; + } + + void rshift_sibling_parent_pos( node_header & node ) noexcept; + void update_right_sibling_link( node_header const & left_node, node_slot left_node_slot ) noexcept; + void unlink_node( node_header & node, node_header & cached_left_sibling ) noexcept; + + [[ gnu::sysv_abi ]] + std::pair new_spillover_node_for( node_header & existing_node ); + + node_placeholder & new_root( node_slot left_child, node_slot right_child ); + template static NodeType & as( SourceNode & slot ) noexcept { @@ -195,18 +258,6 @@ class bptree_base return node.num_vals == node.max_values; } - template - static void umove // uninitialized move - ( - N const & source, node_size_type const srcBegin, node_size_type const srcEnd, - N & target, node_size_type const tgtBegin - ) noexcept - { - BOOST_ASSUME( &source != &target ); // otherwise could require move_backwards or shift_* - BOOST_ASSUME( srcBegin <= srcEnd ); - std::uninitialized_move( &(source.*array)[ srcBegin ], &(source.*array)[ srcEnd ], &(target.*array)[ tgtBegin ] ); - } - [[ nodiscard ]] node_placeholder & new_node(); template @@ -244,7 +295,7 @@ class bptree_base::base_iterator base_iterator( node_pool &, node_slot, node_size_type value_offset ) noexcept; - [[ gnu::pure ]] linked_node_header & node() const noexcept; + [[ gnu::pure ]] node_header & node() const noexcept; public: constexpr base_iterator() noexcept = default; @@ -262,12 +313,11 @@ class bptree_base::base_random_access_iterator : public base_iterator { protected: size_type index_; - node_slot leaves_start_; // merely for the rewind for negative offsets in operator+= template friend class bp_tree; base_random_access_iterator( bptree_base & parent, node_slot const start_leaf, size_type const start_index ) noexcept - : base_iterator{ parent.nodes_, start_leaf, 0 }, index_{ start_index }, leaves_start_{ parent.first_leaf() } {} + : base_iterator{ parent.nodes_, start_leaf, 0 }, index_{ start_index } {} public: constexpr base_random_access_iterator() noexcept = default; @@ -294,11 +344,21 @@ template class bptree_base_wkey : public bptree_base { public: - using key_type = Key; - using value_type = key_type; // TODO map support + using key_type = Key; + using value_type = key_type; // TODO map support using key_const_arg = std::conditional_t, Key, Key const &>; +public: + void reserve( size_type additional_values ) + { + additional_values = additional_values * 3 / 2; // TODO find the appropriate formula + bptree_base::reserve( ( additional_values + leaf_node::max_values - 1 ) / leaf_node::max_values ); + } + + // solely a debugging helper (include b+tree_print.hpp) + void print() const; + protected: // node types struct alignas( node_size ) parent_node : node_header { @@ -319,9 +379,9 @@ class bptree_base_wkey : public bptree_base static node_size_type constexpr max_children{ order }; static node_size_type constexpr max_values { max_children - 1 }; - - node_slot children[ max_children ]; + Key keys [ max_values ]; + node_slot children[ max_children ]; }; // struct inner_node struct inner_node : parent_node @@ -338,12 +398,12 @@ class bptree_base_wkey : public bptree_base static auto constexpr min_values { min_children - 1 }; }; // struct root_node - struct alignas( node_size ) leaf_node : linked_node_header + struct alignas( node_size ) leaf_node : node_header { // TODO support for maps (i.e. keys+values) using value_type = Key; - static node_size_type constexpr storage_space{ node_size - psi::vm::align_up( sizeof( linked_node_header ), alignof( Key ) ) }; + static node_size_type constexpr storage_space{ node_size - psi::vm::align_up( sizeof( node_header ), alignof( Key ) ) }; static node_size_type constexpr max_values { storage_space / sizeof( Key ) }; static node_size_type constexpr min_values { ihalf_ceil }; @@ -367,8 +427,51 @@ class bptree_base_wkey : public bptree_base class fwd_iterator; class ra_iterator; -protected: // helpers for split_to_insert - static value_type insert_into_new_node +protected: // split_to_insert and its helpers + template + void split_to_insert( N & node_to_split, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) + { + auto [ node_slot, new_slot ]{ bptree_base::new_spillover_node_for( node_to_split ) }; + auto const max{ N::max_values }; + auto const mid{ N::min_values }; + auto p_node { &node( node_slot ) }; + auto p_new_node{ &node( new_slot ) }; + verify( *p_node ); + BOOST_ASSUME( p_node->num_vals == max ); + BOOST_ASSERT + ( + !p_node->parent || ( node( p_node->parent ).children[ p_node->parent_child_idx ] == node_slot ) + ); + + auto const new_insert_pos { insert_pos - mid }; + bool const insertion_into_new_node{ new_insert_pos >= 0 }; + auto key_to_propagate{ insertion_into_new_node // we cannot save a reference here because it might get invalidated by the new_node() call below + ? insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) + : insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) + }; + + verify( *p_node ); + verify( *p_new_node ); + BOOST_ASSUME( p_node->num_vals == mid ); + + // propagate the mid key to the parent + if ( p_node->is_root() ) [[ unlikely ]] { + new_root( node_slot, new_slot, std::move( key_to_propagate ) ); + } else { + auto const key_pos{ p_new_node->parent_child_idx /*it is the _right_ child*/ - 1 }; + return insert( node( p_node->parent ), key_pos, std::move( key_to_propagate ), new_slot ); + } + } + + void new_root( node_slot const left_child, node_slot const right_child, key_const_arg separator_key ) + { + auto & new_root_node{ as( bptree_base::new_root( left_child, right_child ) ) }; + new_root_node.keys [ 0 ] = std::move( separator_key ); + new_root_node.children[ 0 ] = left_child; + new_root_node.children[ 1 ] = right_child; + } + + value_type insert_into_new_node ( inner_node & node, inner_node & new_node, key_const_arg value, @@ -391,24 +494,24 @@ class bptree_base_wkey : public bptree_base value_type key_to_propagate; if ( new_insert_pos == 0 ) { - umove<&N::keys >( node, mid , node.num_vals , new_node, 0 ); - umove<&N::children>( node, mid + 1, node.num_vals + 1, new_node, 1 ); - children( new_node )[ new_insert_pos ] = key_right_child; key_to_propagate = std::move( value ); + + move_keys ( node, mid , node.num_vals , new_node, 0 ); + move_chldrn( node, mid + 1, node.num_vals + 1, new_node, 1 ); } else { key_to_propagate = std::move( node.keys[ mid ] ); - umove<&N::keys >( node, mid + 1, insert_pos , new_node, 0 ); - umove<&N::children>( node, mid + 1, insert_pos + 1, new_node, 0 ); + move_keys ( node, mid + 1, insert_pos , new_node, 0 ); + move_chldrn( node, mid + 1, insert_pos + 1, new_node, 0 ); - umove<&N::keys >( node, insert_pos , node.num_vals , new_node, new_insert_pos ); - umove<&N::children>( node, insert_pos + 1, node.num_vals + 1, new_node, new_insert_pos + 1 ); + move_keys ( node, insert_pos , node.num_vals , new_node, new_insert_pos ); + move_chldrn( node, insert_pos + 1, node.num_vals + 1, new_node, new_insert_pos + 1 ); - keys ( new_node )[ new_insert_pos - 1 ] = value; - children( new_node )[ new_insert_pos ] = key_right_child; + keys( new_node )[ new_insert_pos - 1 ] = value; } + insrt_child( new_node, new_insert_pos, key_right_child ); node.num_vals = mid; @@ -435,8 +538,8 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( node.num_vals == max ); BOOST_ASSUME( new_node.num_vals == 0 ); - umove<&N::keys>( node, mid , insert_pos, new_node, 0 ); - umove<&N::keys>( node, insert_pos, max , new_node, new_insert_pos + 1 ); + move_keys( node, mid , insert_pos, new_node, 0 ); + move_keys( node, insert_pos, max , new_node, new_insert_pos + 1 ); node .num_vals = mid ; new_node.num_vals = max - mid + 1; @@ -450,7 +553,7 @@ class bptree_base_wkey : public bptree_base return key_to_propagate; } - static value_type insert_into_existing_node( inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + value_type insert_into_existing_node( inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept { BOOST_ASSUME( bool( key_right_child ) ); @@ -464,17 +567,17 @@ class bptree_base_wkey : public bptree_base value_type key_to_propagate{ std::move( node.keys[ mid - 1 ] ) }; - umove<&N::keys >( node, mid, node.num_vals , new_node, 0 ); - umove<&N::children>( node, mid, node.num_vals + 1, new_node, 0 ); + move_keys ( node, mid, num_vals ( node ), new_node, 0 ); + move_chldrn( node, mid, num_chldrn( node ), new_node, 0 ); - std::shift_right( &node.keys [ insert_pos ], &node.keys [ mid ], 1 ); - std::shift_right( &node.children[ insert_pos + 1 ], &node.children[ mid + 1 ], 1 ); + rshift_keys ( node, insert_pos , mid ); + rshift_chldrn( node, insert_pos + 1, mid + 1 ); node .num_vals = mid; new_node.num_vals = max - mid; - keys ( node )[ insert_pos ] = value; - children( node )[ insert_pos + 1 ] = key_right_child; + keys ( node )[ insert_pos ] = value; + insrt_child( node, insert_pos + 1, key_right_child ); BOOST_ASSUME( !underflowed( node ) ); BOOST_ASSUME( !underflowed( new_node ) ); @@ -494,8 +597,8 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( node.num_vals == max ); BOOST_ASSUME( new_node.num_vals == 0 ); - umove<&N::keys> ( node, mid - 1, max, new_node, 0 ); - std::shift_right( &node.keys[ insert_pos ], &node.keys[ mid ], 1 ); + move_keys( node, mid - 1 , max, new_node, 0 ); + rshift_keys( node, insert_pos, mid ); node .num_vals = mid; new_node.num_vals = max - mid + 1; @@ -510,80 +613,278 @@ class bptree_base_wkey : public bptree_base } protected: // 'other' + template + void insert( N & target_node, node_size_type const target_node_pos, key_const_arg v, node_slot const right_child ) + { + verify( target_node ); + if ( full( target_node ) ) [[ unlikely ]] { + return split_to_insert( target_node, target_node_pos, v, right_child ); + } else { + ++target_node.num_vals; + rshift_keys( target_node, target_node_pos ); + target_node.keys[ target_node_pos ] = v; + if constexpr ( requires { target_node.children; } ) { + auto const ch_pos{ target_node_pos + /*>right< child*/ 1 }; + rshift_chldrn( target_node, ch_pos ); + this->insrt_child( target_node, ch_pos, right_child ); + } + } + } + + template + BOOST_NOINLINE + void handle_underflow( N & node, depth_t const level ) + { + BOOST_ASSUME( underflowed( node ) ); + + auto constexpr parent_node_type{ requires{ node.children; } }; + auto constexpr leaf_node_type { !parent_node_type }; + + BOOST_ASSUME( level > 0 ); + BOOST_ASSUME( !leaf_node_type || level == leaf_level() ); + auto const node_slot{ slot_of( node ) }; + auto & parent{ this->node( node.parent ) }; + verify( parent ); + + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); + auto const parent_child_idx { node.parent_child_idx }; + bool const parent_has_key_copy{ leaf_node_type && ( parent_child_idx > 0 ) }; + auto const parent_key_idx { parent_child_idx - parent_has_key_copy }; + BOOST_ASSUME( !parent_has_key_copy || parent.keys[ parent_key_idx ] == node.keys[ 0 ] ); + + BOOST_ASSUME( parent.children[ parent_child_idx ] == node_slot ); + // the left and right level dlink pointers can point 'across' parents + // (and so cannot be used to resolve siblings) + auto const right_separator_key_idx{ static_cast( parent_key_idx + parent_has_key_copy ) }; + auto const left_separator_key_idx{ std::min( static_cast( right_separator_key_idx - 1 ), parent.num_vals ) }; // (ab)use unsigned wraparound + + auto const has_right_sibling{ parent_child_idx < ( num_chldrn( parent ) - 1 ) }; + auto const has_left_sibling { parent_child_idx > 0 }; + auto const p_right_sibling{ has_right_sibling ? &this->node( node.right ) : nullptr }; + auto const p_left_sibling { has_left_sibling ? &this->node( node.left ) : nullptr }; + + auto const p_right_separator_key{ has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; + auto const p_left_separator_key { has_left_sibling ? &parent.keys[ left_separator_key_idx ] : nullptr }; + BOOST_ASSUME( has_right_sibling || has_left_sibling ); + BOOST_ASSERT( &node != p_left_sibling ); + BOOST_ASSERT( &node != p_right_sibling ); + BOOST_ASSERT( static_cast( &node ) != static_cast( &parent ) ); + // Borrow from left sibling if possible + if ( p_left_sibling && can_borrow( *p_left_sibling ) ) + { + verify( *p_left_sibling ); + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); + node.num_vals++; + rshift_keys( node ); + BOOST_ASSUME( node.num_vals == node.min_values ); + auto const node_keys{ keys( node ) }; + auto const left_keys{ keys( *p_left_sibling ) }; + + if constexpr ( leaf_node_type ) { + // Move the largest key from left sibling to the current node + BOOST_ASSUME( parent_has_key_copy ); + node_keys.front() = std::move( left_keys.back() ); + // adjust the separator key in the parent + BOOST_ASSERT( *p_left_separator_key == node_keys[ 1 ] ); + *p_left_separator_key = node_keys.front(); + } else { + // Move/rotate the largest key from left sibling to the current node 'through' the parent + BOOST_ASSERT( left_keys.back() < *p_left_separator_key ); + BOOST_ASSERT( *p_left_separator_key < node_keys.front() ); + node_keys.front() = std::move( *p_left_separator_key ); + *p_left_separator_key = std::move( left_keys.back() ); + + rshift_chldrn( node ); + insrt_child( node, 0, children( *p_left_sibling ).back(), node_slot ); + } + + p_left_sibling->num_vals--; + verify( *p_left_sibling ); + + BOOST_ASSUME( node. num_vals == N::min_values ); + BOOST_ASSUME( p_left_sibling->num_vals >= N::min_values ); + } + // Borrow from right sibling if possible + else + if ( p_right_sibling && can_borrow( *p_right_sibling ) ) + { + verify( *p_right_sibling ); + BOOST_ASSUME( node.num_vals == node.min_values - 1 ); + node.num_vals++; + auto const node_keys{ keys( node ) }; + if constexpr ( leaf_node_type ) { + // Move the smallest key from the right sibling to the current node + auto & leftmost_right_key{ keys( *p_right_sibling ).front() }; + BOOST_ASSUME( *p_right_separator_key == leftmost_right_key ); + node_keys.back() = std::move( leftmost_right_key ); + lshift_keys( *p_right_sibling ); + // adjust the separator key in the parent + *p_right_separator_key = leftmost_right_key; + } else { + // Move/rotate the smallest key from the right sibling to the current node 'through' the parent + BOOST_ASSUME( parent.keys[ parent_child_idx ] < p_right_sibling->keys[ 0 ] ); + BOOST_ASSERT( *( node_keys.end() - 2 ) < *p_right_separator_key ); + node_keys.back() = std::move( *p_right_separator_key ); + *p_right_separator_key = std::move( keys( *p_right_sibling ).front() ); + insrt_child( node, num_chldrn( node ) - 1, children( *p_right_sibling ).front(), node_slot ); + lshift_keys ( *p_right_sibling ); + lshift_chldrn( *p_right_sibling ); + } + + p_right_sibling->num_vals--; + verify( *p_right_sibling ); + + BOOST_ASSUME( node. num_vals == N::min_values ); + BOOST_ASSUME( p_right_sibling->num_vals >= N::min_values ); + } + // Merge with left or right sibling + else + { + if ( p_left_sibling ) { + verify( *p_left_sibling ); + BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); + // Merge node -> left sibling + this->merge_right_into_left( *p_left_sibling, node, parent, left_separator_key_idx, parent_child_idx ); + } else { + verify( *p_right_sibling ); + BOOST_ASSUME( parent_key_idx == 0 ); + BOOST_ASSUME( parent.keys[ parent_key_idx ] <= p_right_sibling->keys[ 0 ] ); + // Merge right sibling -> node + this->merge_right_into_left( node, *p_right_sibling, parent, right_separator_key_idx, static_cast( parent_child_idx + 1 ) ); + } + + // propagate underflow + auto & __restrict depth_{ this->hdr().depth_ }; + auto & __restrict root_ { this->hdr().root_ }; + if ( parent.is_root() ) [[ unlikely ]] + { + BOOST_ASSUME( root_ == slot_of( parent ) ); + BOOST_ASSUME( level == 1 ); + BOOST_ASSUME( depth_ > level ); // 'leaf root' should be handled directly by the special case in erase() + auto & root{ as( parent ) }; + BOOST_ASSUME( !!root.children[ 0 ] ); + if ( underflowed( root ) ) + { + root_ = root.children[ 0 ]; + bptree_base::node( root_ ).parent = {}; + --depth_; + free( root ); + } + } + else + if ( underflowed( parent ) ) + { + BOOST_ASSUME( level > 0 ); + BOOST_ASSUME( level < depth_ ); + handle_underflow( parent, level - 1 ); + } + } + } // handle_underflow() + root_node & root() noexcept { return as( bptree_base::root() ); } root_node const & root() const noexcept { return const_cast( *this ).root(); } using bptree_base::free; - void free( leaf_node & node ) noexcept { + void free( leaf_node & leaf ) noexcept { auto & first_leaf{ hdr().leaves_ }; - if ( first_leaf == slot_of( node ) ) { - first_leaf = node.next; + if ( first_leaf == slot_of( leaf ) ) { + BOOST_ASSUME( !leaf.left ); + first_leaf = leaf.right; + if ( first_leaf ) + { + BOOST_ASSERT( node( first_leaf ).left ); + node( first_leaf ).left = {}; + } } - bptree_base::free( static_cast( node ) ); + bptree_base::free( static_cast( leaf ) ); + } + + template + [[ gnu::sysv_abi ]] static + void move_keys + ( + N const & source, node_size_type src_begin, node_size_type src_end, + N & target, node_size_type tgt_begin + ) noexcept; + [[ gnu::sysv_abi ]] + void move_chldrn + ( + inner_node const & source, node_size_type src_begin, node_size_type src_end, + inner_node & target, node_size_type tgt_begin + ) noexcept; + + void insrt_child( inner_node & target, node_size_type const pos, node_slot const child_slot, node_slot const cached_target_slot ) noexcept + { + BOOST_ASSUME( cached_target_slot == slot_of( target ) ); + auto & child{ node( child_slot ) }; + children( target )[ pos ] = child_slot; + child.parent = cached_target_slot; + child.parent_child_idx = pos; + } + void insrt_child( inner_node & target, node_size_type const pos, node_slot const child_slot ) noexcept + { + insrt_child( target, pos, child_slot, slot_of( target ) ); } - void merge_nodes + [[ gnu::sysv_abi ]] + void merge_right_into_left ( - leaf_node & __restrict source, leaf_node & __restrict target, + leaf_node & __restrict left, leaf_node & __restrict right, inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx ) noexcept { auto constexpr min{ leaf_node::min_values }; - BOOST_ASSUME( source.num_vals >= min - 1 ); BOOST_ASSUME( source.num_vals <= min ); - BOOST_ASSUME( target.num_vals >= min - 1 ); BOOST_ASSUME( target.num_vals <= min ); - - std::ranges::move( keys ( source ), keys ( target ).end() ); - std::ranges::move( children( source ), children( target ).end() ); - target.num_vals += source.num_vals; - source.num_vals = 0; - target.next = source.next; - BOOST_ASSUME( target.num_vals >= 2 * min - 2 ); - BOOST_ASSUME( target.num_vals <= 2 * min - 1 ); - std::ranges::shift_left( keys ( parent ).subspan( parent_key_idx ), 1 ); - std::ranges::shift_left( children( parent ).subspan( parent_child_idx ), 1 ); + BOOST_ASSUME( right.num_vals >= min - 1 ); BOOST_ASSUME( right.num_vals <= min ); + BOOST_ASSUME( left .num_vals >= min - 1 ); BOOST_ASSUME( left .num_vals <= min ); + BOOST_ASSUME( left .right == slot_of( right ) ); + BOOST_ASSUME( right.left == slot_of( left ) ); + + std::ranges::move( keys( right ), keys( left ).end() ); + left .num_vals += right.num_vals; + right.num_vals = 0; + BOOST_ASSUME( left.num_vals >= 2 * min - 2 ); + BOOST_ASSUME( left.num_vals <= 2 * min - 1 ); + lshift_keys ( parent, parent_key_idx ); + lshift_chldrn( parent, parent_child_idx ); BOOST_ASSUME( parent.num_vals ); parent.num_vals--; - source.next = slot_of( target ); - verify( target ); - free ( source ); + unlink_node( right, left ); + + right.right = {}; + verify( left ); + free ( right ); verify( parent ); } - - void merge_nodes + [[ gnu::sysv_abi ]] + void merge_right_into_left ( - inner_node & __restrict right, inner_node & __restrict left, + inner_node & __restrict left, inner_node & __restrict right, inner_node & __restrict parent, node_size_type const parent_key_idx, node_size_type const parent_child_idx ) noexcept { auto constexpr min{ inner_node::min_values }; BOOST_ASSUME( right.num_vals >= min - 1 ); BOOST_ASSUME( right.num_vals <= min ); BOOST_ASSUME( left .num_vals >= min - 1 ); BOOST_ASSUME( left .num_vals <= min ); + unlink_node( right, left ); - for ( auto const ch_slot : children( right ) ) - this->node( ch_slot ).parent = slot_of( left ); - - std::ranges::move( children( right ), children( left ).end() ); - left.num_vals += 1; + move_chldrn( right, 0, num_chldrn( right ), left, num_chldrn( left ) ); auto & separator_key{ parent.keys[ parent_key_idx ] }; + left.num_vals += 1; keys( left ).back() = std::move( separator_key ); std::ranges::move( keys( right ), keys( left ).end() ); left .num_vals += right.num_vals; right.num_vals = 0; - BOOST_ASSUME( left.num_vals == left.max_values ); + BOOST_ASSUME( left.num_vals >= left.max_values - 1 ); BOOST_ASSUME( left.num_vals <= left.max_values ); verify( left ); free ( right ); - std::ranges::shift_left( keys ( parent ).subspan( parent_key_idx ), 1 ); - std::ranges::shift_left( children( parent ).subspan( parent_child_idx ), 1 ); + lshift_keys ( parent, parent_key_idx ); + lshift_chldrn( parent, parent_child_idx ); BOOST_ASSUME( parent.num_vals ); parent.num_vals--; } - -public: - // solely a debugging helper - void print() const; }; // class bptree_base_wkey //////////////////////////////////////////////////////////////////////////////// @@ -638,6 +939,45 @@ class bptree_base_wkey::ra_iterator operator fwd_iterator() const noexcept { return static_cast( static_cast( *this ) ); } }; // class ra_iterator +template +template [[ gnu::noinline, gnu::sysv_abi ]] +void bptree_base_wkey::move_keys +( + N const & source, node_size_type const src_begin, node_size_type const src_end, + N & target, node_size_type const tgt_begin +) noexcept +{ + BOOST_ASSUME( &source != &target ); // otherwise could require move_backwards or shift_* + BOOST_ASSUME( src_begin <= src_end ); + BOOST_ASSUME( ( src_end - src_begin ) <= N::min_values + 1 ); + BOOST_ASSUME( tgt_begin < N::max_values ); + std::uninitialized_move( &source.keys[ src_begin ], &source.keys[ src_end ], &target.keys[ tgt_begin ] ); +} +template [[ gnu::noinline, gnu::sysv_abi ]] +void bptree_base_wkey::move_chldrn +( + inner_node const & source, node_size_type const src_begin, node_size_type const src_end, + inner_node & target, node_size_type const tgt_begin +) noexcept +{ + BOOST_ASSUME( &source != &target ); // otherwise could require move_backwards or shift_* + BOOST_ASSUME( src_begin <= src_end ); + auto const count{ static_cast( src_end - src_begin ) }; + BOOST_ASSUME( count <= inner_node::min_children + 1 ); + BOOST_ASSUME( tgt_begin < inner_node::max_children ); + auto const src_chldrn{ &source.children[ src_begin ] }; + + auto const target_slot{ slot_of( target ) }; + for ( auto ch{ 0 }; ch < count; ++ch ) + { + auto & ch_slot{ src_chldrn[ ch ] }; + auto & child { node( ch_slot ) }; + target.children[ tgt_begin + ch ] = std::move( ch_slot ); + child.parent = target_slot; + child.parent_child_idx = tgt_begin + ch; + } +} + //////////////////////////////////////////////////////////////////////////////// // \class bp_tree @@ -673,11 +1013,16 @@ class bp_tree using bptree_base::keys; using bptree_base::node; using bptree_base::num_chldrn; + using bptree_base::lshift_keys; + using bptree_base::rshift_keys; + using bptree_base::lshift_chldrn; + using bptree_base::rshift_chldrn; using bptree_base::slot_of; using bptree_base::underflowed; using bptree_base::verify; using base::free; + using base::insrt_child; using base::leaf_level; using base::root; @@ -745,9 +1090,10 @@ class bp_tree BOOST_NOINLINE void erase( key_const_arg key ) noexcept { - auto & __restrict depth_{ base::hdr().depth_ }; - auto & __restrict root_ { base::hdr().root_ }; - auto & __restrict size_ { base::hdr().size_ }; + auto & hdr{ base::hdr() }; + auto & __restrict depth_{ hdr.depth_ }; + auto & __restrict root_ { hdr.root_ }; + auto & __restrict size_ { hdr.size_ }; auto const nodes{ find_nodes_for( key ) }; @@ -758,14 +1104,14 @@ class bp_tree BOOST_ASSUME( leaf.num_vals >= leaf.min_values ); } auto const key_offset{ find( leaf, key ) }; - + // code below can go into base if ( nodes.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" { BOOST_ASSUME( key_offset == 0 ); BOOST_ASSUME( leaf.keys[ key_offset ] == key ); - auto * p_node{ &node( nodes.inner ) }; - auto & separator_key{ p_node->keys[ nodes.inner_offset ] }; + auto & inner { node( nodes.inner ) }; + auto & separator_key{ inner.keys[ nodes.inner_offset ] }; BOOST_ASSUME( separator_key == key ); BOOST_ASSUME( key_offset + 1 < leaf.num_vals ); separator_key = leaf.keys[ key_offset + 1 ]; @@ -773,7 +1119,7 @@ class bp_tree BOOST_ASSUME( key_offset != leaf.num_vals && leaf.keys[ key_offset ] == key ); - std::ranges::shift_left( keys( leaf ).subspan( key_offset ), 1 ); + lshift_keys( leaf, key_offset ); BOOST_ASSUME( leaf.num_vals ); --leaf.num_vals; if ( depth_ == 1 ) [[ unlikely ]] // handle 'leaf root' deletion directly to simplify handle_underflow() @@ -781,6 +1127,8 @@ class bp_tree BOOST_ASSUME( root_ == slot_of( leaf ) ); BOOST_ASSUME( leaf.is_root() ); BOOST_ASSUME( size_ == leaf.num_vals + 1 ); + BOOST_ASSUME( !leaf.left ); + BOOST_ASSUME( !leaf.right ); if ( leaf.num_vals == 0 ) { root_ = {}; @@ -793,14 +1141,12 @@ class bp_tree { BOOST_ASSUME( !leaf.is_root() ); BOOST_ASSUME( depth_ > 1 ); - handle_underflow( leaf, base::leaf_level() ); + base::handle_underflow( leaf, base::leaf_level() ); } --size_; } - void reserve( size_type const additional_values ) { bptree_base::reserve( ( additional_values + leaf_node::max_values - 1 ) / leaf_node::max_values ); } - void swap( bp_tree & other ) noexcept { base::swap( other ); } Comparator const & comp() const noexcept { return *this; } @@ -822,243 +1168,13 @@ class bp_tree return find( node.keys, node.num_vals, value ); } - template - void split_to_insert( N * p_node, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) - { - verify( *p_node ); - - auto const max{ N::max_values }; - auto const mid{ N::min_values }; - - auto const node_slot{ slot_of( *p_node ) }; - auto * p_new_node{ &bptree_base::new_node() }; - auto const new_slot{ slot_of( *p_new_node ) }; - p_node = &bptree_base::node( node_slot ); // handle relocation - BOOST_ASSUME( p_node->num_vals == max ); - - auto constexpr parent_node_type{ requires{ p_new_node->children; } }; - - p_new_node->parent = p_node->parent; - if constexpr ( !parent_node_type ) - { - // leaf linked list - // p_node <- left/lesser | new_node -> right/greater - p_new_node->next = p_node->next; - p_node ->next = new_slot; - } - - auto const new_insert_pos { insert_pos - mid }; - bool const insertion_into_new_node{ new_insert_pos >= 0 }; - auto key_to_propagate{ insertion_into_new_node // we cannnot save a reference here because it might get invalidated by the new_node() call below - ? base::insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) - : base::insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) - }; - - verify( *p_node ); - verify( *p_new_node ); - BOOST_ASSUME( p_node->num_vals == mid ); - - if constexpr ( parent_node_type ) { - for ( auto const ch_slot : children( *p_new_node ) ) { - node( ch_slot ).parent = new_slot; - } - } - - // propagate the mid key to the parent - if ( p_node->is_root() ) [[ unlikely ]] { - auto & newRoot{ bptree_base::new_node() }; - auto & hdr { this->hdr() }; - p_node = &node( node_slot ); - p_new_node = &node( new_slot ); - newRoot.keys [ 0 ] = std::move( key_to_propagate ); - newRoot.children[ 0 ] = node_slot; - newRoot.children[ 1 ] = new_slot; - newRoot.num_vals = 1; - hdr.root_ = slot_of( newRoot ); - p_node ->parent = hdr.root_; - p_new_node->parent = hdr.root_; - ++hdr.depth_; - } else { - return insert( node( p_node->parent ), key_to_propagate, new_slot ); - } - } - template void insert( N & target_node, key_const_arg v, node_slot const right_child ) { - verify( target_node ); auto const pos{ find( target_node, v ) }; - if ( base::full( target_node ) ) [[ unlikely ]] { - return split_to_insert( &target_node, pos, v, right_child ); - } else { - ++target_node.num_vals; - std::ranges::shift_right( base::keys( target_node ).subspan( pos ), 1 ); - target_node.keys[ pos ] = v; - if constexpr ( requires { target_node.children; } ) { - std::ranges::shift_right( children( target_node ).subspan( pos + /*>right< child*/1 ), 1 ); - target_node.children[ pos + 1 ] = right_child; - } - } + base::insert( target_node, pos, v, right_child ); } - - template - BOOST_NOINLINE - void handle_underflow( N & node, depth_t const level ) - { - auto & __restrict depth_{ this->hdr().depth_ }; - auto & __restrict root_ { this->hdr().root_ }; - - BOOST_ASSUME( underflowed( node ) ); - - auto constexpr parent_node_type{ requires{ node.children; } }; - auto constexpr leaf_node_type { !parent_node_type }; - - BOOST_ASSUME( level > 0 ); - if constexpr ( leaf_node_type ) { - BOOST_ASSUME( level == leaf_level() ); - } - auto const node_slot{ slot_of( node ) }; - auto & parent{ this->node( node.parent ) }; - verify( parent ); - - BOOST_ASSUME( node.num_vals == node.min_values - 1 ); - - auto const parent_key_idx { find( parent, node.keys[ 0 ] ) }; BOOST_ASSUME( parent_key_idx != parent.num_vals || !leaf_node_type ); - bool const parent_has_key_copy{ leaf_node_type && ( parent.keys[ parent_key_idx ] == node.keys[ 0 ] ) }; - auto const parent_child_idx { static_cast( parent_key_idx + parent_has_key_copy ) }; - BOOST_ASSUME( parent.children[ parent_child_idx ] == node_slot ); - - auto const right_separator_key_idx{ static_cast( parent_key_idx + parent_has_key_copy ) }; - auto const left_separator_key_idx{ std::min( static_cast( right_separator_key_idx - 1 ), parent.num_vals ) }; // (ab)use unsigned wraparound - auto const has_right_sibling { right_separator_key_idx != parent.num_vals }; - auto const has_left_sibling { left_separator_key_idx != parent.num_vals }; - auto const p_right_separator_key { has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; - auto const p_left_separator_key { has_left_sibling ? &parent.keys[ left_separator_key_idx ] : nullptr }; - BOOST_ASSUME( has_right_sibling == ( parent_child_idx < num_chldrn( parent ) - 1 ) ); - BOOST_ASSUME( has_left_sibling == ( parent_child_idx > 0 ) ); - auto const p_right_sibling{ has_right_sibling ? &this->node( children( parent )[ parent_child_idx + 1 ] ) : nullptr }; - auto const p_left_sibling { has_left_sibling ? &this->node( children( parent )[ parent_child_idx - 1 ] ) : nullptr }; - - BOOST_ASSERT( &node != p_left_sibling ); - BOOST_ASSERT( &node != p_right_sibling ); - BOOST_ASSERT( static_cast( &node ) != static_cast( &parent ) ); - // Borrow from left sibling if possible - if ( p_left_sibling && can_borrow( *p_left_sibling ) ) - { - verify( *p_left_sibling ); - BOOST_ASSUME( node.num_vals == node.min_values - 1 ); - node.num_vals++; - BOOST_ASSUME( node.num_vals == node.min_values ); - - auto const node_vals{ keys( node ) }; - auto const left_vals{ keys( *p_left_sibling ) }; - std::ranges::shift_right( node_vals, 1 ); - - if constexpr ( leaf_node_type ) { - // Move the largest key from left sibling to the current node - BOOST_ASSUME( parent_has_key_copy ); - node_vals.front() = std::move( left_vals.back() ); - // adjust the separator key in the parent - BOOST_ASSERT( *p_left_separator_key == node_vals[ 1 ] ); - *p_left_separator_key = node_vals.front(); - } else { - // Move/rotate the largest key from left sibling to the current node 'through' the parent - BOOST_ASSERT( left_vals.back() < *p_left_separator_key ); - BOOST_ASSERT( *p_left_separator_key < node_vals.front() ); - node_vals.front() = std::move( *p_left_separator_key ); - *p_left_separator_key = std::move( left_vals.back() ); - - auto const chldrn{ children( node ) }; - std::ranges::shift_right( chldrn, 1 ); - chldrn.front() = std::move( children( *p_left_sibling ).back() ); - this->node( chldrn.front() ).parent = node_slot; - } - - p_left_sibling->num_vals--; - verify( *p_left_sibling ); - - BOOST_ASSUME( node. num_vals == N::min_values ); - BOOST_ASSUME( p_left_sibling->num_vals >= N::min_values ); - } - // Borrow from right sibling if possible - else - if ( p_right_sibling && can_borrow( *p_right_sibling ) ) - { - verify( *p_right_sibling ); - - node.num_vals++; - BOOST_ASSUME( node.num_vals == node.min_values ); - - if constexpr ( leaf_node_type ) { - // Move the smallest key from right sibling to current node the parent - auto right_vals{ keys( *p_right_sibling ) }; - BOOST_ASSERT( parent.keys[ parent_child_idx ] == right_vals[ 0 ] ); - keys( node ).back() = std::move( right_vals.front() ); - std::ranges::shift_left( right_vals, 1 ); - // adjust the separator key in the parent - *p_right_separator_key = right_vals.front(); - } else { - // Move/rotate the smallest key from right sibling to current node 'through' the parent - BOOST_ASSUME( parent.keys[ parent_child_idx ] < p_right_sibling->keys[ 0 ] ); - BOOST_ASSERT( *( keys( node ).end() - 2 ) < *p_right_separator_key ); - keys( node ).back() = std::move( *p_right_separator_key ); - *p_right_separator_key = std::move( keys ( *p_right_sibling ).front() ); - children( node ).back() = std::move( children( *p_right_sibling ).front() ); - std::ranges::shift_left( keys ( *p_right_sibling ), 1 ); - std::ranges::shift_left( children( *p_right_sibling ), 1 ); - this->node( children( node ).back() ).parent = node_slot; - } - - p_right_sibling->num_vals--; - verify( *p_right_sibling ); - - BOOST_ASSUME( node. num_vals == N::min_values ); - BOOST_ASSUME( p_right_sibling->num_vals >= N::min_values ); - } - // Merge with left or right sibling - else - { - if ( p_left_sibling ) { - verify( *p_left_sibling ); - BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); - // Merge node -> left sibling - this->merge_nodes( node, *p_left_sibling, parent, left_separator_key_idx, parent_child_idx ); - } else { - verify( *p_right_sibling ); - BOOST_ASSUME( parent_key_idx == 0 ); - BOOST_ASSUME( parent.keys[ parent_key_idx ] <= p_right_sibling->keys[ 0 ] ); - // Merge right sibling -> node - this->merge_nodes( *p_right_sibling, node, parent, right_separator_key_idx, static_cast( parent_child_idx + 1 ) ); - } - - // propagate underflow - if ( parent.is_root() ) [[ unlikely ]] - { - BOOST_ASSUME( root_ == slot_of( parent ) ); - BOOST_ASSUME( level == 1 ); - BOOST_ASSUME( depth_ > level ); // 'leaf root' should be handled directly by the special case in erase() - auto & root{ as( parent ) }; - BOOST_ASSUME( !!root.children[ 0 ] ); - if ( underflowed( root ) ) - { - root_ = root.children[ 0 ]; - bptree_base::node( root_ ).parent = {}; - --depth_; - free( root ); - } - } - else - if ( underflowed( parent ) ) - { - BOOST_ASSUME( level > 0 ); - BOOST_ASSUME( level < depth_ ); - handle_underflow( parent, level - 1 ); - } - } - } // handle_underflow() - - struct key_locations { leaf_node & leaf; @@ -1075,11 +1191,13 @@ class bp_tree // if the root is a (lone) leaf_node is implicitly handled by the loop condition: // depth_ == 1 so the loop is skipped entirely and the lone root is never examined // through the incorrectly typed reference - auto * p_node{ &bptree_base::as( root() ) }; - for ( auto level{ 0 }; level < this->hdr().depth_ - 1; ++level ) + auto p_node{ &bptree_base::as( root() ) }; + auto const depth { this->hdr().depth_ }; + for ( auto level{ 0 }; level < depth - 1; ++level ) { auto pos{ find( *p_node, key ) }; - if ( pos != p_node->num_vals && p_node->keys[ pos ] == key ) { + if ( pos != p_node->num_vals && p_node->keys[ pos ] == key ) + { // separator key - it also means we have to traverse to the right BOOST_ASSUME( !separator_key_node ); separator_key_node = slot_of( *p_node ); diff --git a/include/psi/vm/containers/b+tree_print.hpp b/include/psi/vm/containers/b+tree_print.hpp index 5d65ab3..10f27fa 100644 --- a/include/psi/vm/containers/b+tree_print.hpp +++ b/include/psi/vm/containers/b+tree_print.hpp @@ -2,8 +2,8 @@ #include "b+tree.hpp" +#include #include -#include //------------------------------------------------------------------------------ namespace psi::vm { @@ -18,62 +18,63 @@ void bptree_base_wkey::print() const return; } - // Queue for performing level-order traversal (BFS). - std::queue nodes; - nodes.push( &as( root() ) ); - - // Current level tracker - depth_t level{ 0 }; - - // Perform BFS, processing one level of the tree at a time. - while ( !nodes.empty() ) + // BFS, one level of the tree at a time. + auto p_node{ &as( root() ) }; + for ( auto level{ 0U }; !is_leaf_level( level ); ++level ) { - auto node_count{ nodes.size() }; - std::print( "Level {} ({} nodes):\n\t", std::uint16_t( level ), node_count ); + std::print( "Level {}:\t", std::uint16_t( level ) ); - size_t level_key_count{ 0 }; + auto p_next_level{ &node( children( *p_node ).front() ) }; + + std::uint32_t level_node_count{ 0 }; + size_type level_key_count { 0 }; // Process all nodes at the current level - while ( node_count > 0 ) + for ( ; ; ) { - auto const node{ nodes.front() }; - nodes.pop(); - - if ( is_leaf_level( level ) ) + // Internal node, print keys and add children to the queue + level_key_count += num_vals( *p_node ); + std::putchar( '<' ); + for ( auto i{ 0U }; i < num_vals( *p_node ); ++i ) { - auto & ln{ as( *node ) }; - level_key_count += num_vals( ln ); - std::putchar( '[' ); - for ( auto i{ 0U }; i < num_vals( ln ); ++i ) - { - std::print( "{}", keys( ln )[ i ] ); - if ( i < num_vals( ln ) - 1U ) - std::print( ", " ); - } - std::print( "] " ); + std::print( "{}", keys( *p_node )[ i ] ); + if ( i < num_vals( *p_node ) - 1U ) + std::print( ", " ); } - else - { - // Internal node, print keys and add children to the queue - level_key_count += num_vals( *node ); - std::putchar( '<' ); - for ( auto i{ 0U }; i < num_vals( *node ); ++i ) - { - std::print( "{}", keys( *node )[ i ] ); - if ( i < num_vals( *node ) - 1U ) - std::print( ", " ); - } - std::print( "> " ); + std::print( "> " ); - // Add all children of this internal node to the queue - for ( auto i{ 0 }; i < num_chldrn( *node ); ++i ) - nodes.push( &this->node( node->children[ i ] ) ); + ++level_node_count; + if ( !p_node->right ) + break; + p_node = &node( p_node->right ); + } + std::println( " [{} nodes w/ {} values]", level_node_count, level_key_count ); + + p_node = p_next_level; + } + + { + std::print( "Leaf level ({}):\t", leaf_level() ); + std::uint32_t level_node_count{ 0 }; + size_type level_key_count { 0 }; + auto p_leaf{ &as( *p_node ) }; + for ( ; ; ) + { + level_key_count += num_vals( *p_leaf ); + std::putchar( '[' ); + for ( auto i{ 0U }; i < num_vals( *p_leaf ); ++i ) { + std::print( "{}", keys( *p_leaf )[ i ] ); + if ( i < num_vals( *p_leaf ) - 1U ) { + std::print( ", " ); + } } + std::print( "] " ); - --node_count; + ++level_node_count; + if ( !p_leaf->right ) + break; + p_leaf = &node( p_leaf->right ); } - - std::println( " [{} values]", level_key_count ); - ++level; + std::println( " [{} nodes w/ {} values]", level_node_count, level_key_count ); } } diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index d2e8e48..374c29e 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -52,6 +52,80 @@ void bptree_base::reserve( node_slot::value_type const additional_nodes ) free( n ); } +void bptree_base::rshift_sibling_parent_pos( node_header & node ) noexcept +{ + auto p_node{ &node }; + while ( p_node->right ) + { + auto & right{ this->node( p_node->right ) }; + BOOST_ASSUME( right.parent_child_idx == p_node->parent_child_idx ); + ++right.parent_child_idx; + p_node = &right; + } +} + +void bptree_base::update_right_sibling_link( node_header const & left_node, node_slot const left_node_slot ) noexcept +{ + BOOST_ASSUME( slot_of( left_node ) == left_node_slot ); + if ( left_node.right ) [[ likely ]] + { + auto & right_back_left{ this->node( left_node.right ).left }; + BOOST_ASSUME( right_back_left != left_node_slot ); + right_back_left = left_node_slot; + } +} + +void bptree_base::unlink_node( node_header & node, node_header & cached_left_sibling ) noexcept +{ + auto & left { cached_left_sibling }; + auto & right{ node }; + BOOST_ASSUME( left .right == slot_of( right ) ); + BOOST_ASSUME( right.left == slot_of( left ) ); + left.right = right.right; + update_right_sibling_link( left, right.left ); +} + +[[ gnu::noinline, gnu::sysv_abi ]] +std::pair +bptree_base::new_spillover_node_for( node_header & existing_node ) +{ + auto const existing_node_slot{ slot_of( existing_node ) }; + auto & right_node { new_node() }; + auto & left_node { node( existing_node_slot ) }; // handle relocation (by new_node()) + auto const right_node_slot { slot_of( right_node ) }; + + // insert into the level dlinked list + // node <- left/lesser | new_node -> right/greater + right_node.left = existing_node_slot; + right_node.right = left_node.right; + left_node.right = right_node_slot; + update_right_sibling_link( right_node, right_node_slot ); + right_node.parent = left_node.parent; + right_node.parent_child_idx = left_node.parent_child_idx + 1; + //right-sibling parent_child_idx rshifting is performed by insert_into_* + //rshift_sibling_parent_pos( right_node ); + + return std::make_pair( existing_node_slot, right_node_slot ); +} +[[ gnu::noinline ]] +bptree_base::node_placeholder & +bptree_base::new_root( node_slot const left_child, node_slot const right_child ) +{ + auto & new_root{ new_node() }; + auto & hdr { this->hdr() }; + auto & left { node( left_child ) }; BOOST_ASSUME( left.is_root() ); + auto & right{ node( right_child ) }; + new_root.left = new_root.right = {}; + new_root.num_vals = 1; + hdr.root_ = slot_of( new_root ); + left .parent = hdr.root_; + right.parent = hdr.root_; + BOOST_ASSUME( left .parent_child_idx == 0 ); + BOOST_ASSUME( right.parent_child_idx == 1 ); + ++hdr.depth_; + return new_root; +} + bptree_base::base_iterator::base_iterator( node_pool & nodes, node_slot const node_offset, node_size_type const value_offset ) noexcept : #ifndef NDEBUG // for bounds checking @@ -62,8 +136,8 @@ bptree_base::base_iterator::base_iterator( node_pool & nodes, node_slot const no node_slot_{ node_offset }, value_offset_{ value_offset } {} -bptree_base::linked_node_header & -bptree_base::base_iterator::node() const noexcept { return static_cast( static_cast( nodes_[ *node_slot_ ] ) ); } +bptree_base::node_header & +bptree_base::base_iterator::node() const noexcept { return nodes_[ *node_slot_ ]; } bptree_base::base_iterator & bptree_base::base_iterator::operator++() noexcept @@ -73,7 +147,7 @@ bptree_base::base_iterator::operator++() noexcept BOOST_ASSUME( node.num_vals >= 1 ); if ( ++value_offset_ == node.num_vals ) [[ unlikely ]] { // implicitly becomes an end iterator when node.next is 'null' - node_slot_ = node.next; + node_slot_ = node.right; value_offset_ = 0; } return *this; @@ -89,33 +163,48 @@ bool bptree_base::base_iterator::operator==( base_iterator const & other ) const bptree_base::base_random_access_iterator & bptree_base::base_random_access_iterator::operator+=( difference_type const n ) noexcept { - if ( n < 0 ) + BOOST_ASSERT_MSG( node_slot_, "Iterator at end: not incrementable" ); + + if ( n >= 0 ) [[ likely ]] { - auto const un{ static_cast( -n ) }; - BOOST_ASSUME( un < index_ ); - auto const absolute_pos{ index_ - un }; - node_slot_ = leaves_start_; - value_offset_ = 0; - index_ = 0; - return *this += static_cast( absolute_pos ); + auto un{ static_cast( n ) }; + for ( ;; ) + { + auto & node{ this->node() }; + auto const available_offset{ static_cast( node.num_vals - 1 - value_offset_ ) }; + if ( available_offset >= un ) [[ likely ]] { + value_offset_ += static_cast( un ); + break; + } else { + un -= available_offset; + node_slot_ = node.right; + value_offset_ = 0; + BOOST_ASSERT_MSG( node_slot_ || un == 0, "Incrementing out of bounds" ); + } + } + index_ += static_cast( n ); } - - BOOST_ASSERT_MSG( node_slot_, "Iterator at end: not incrementable" ); - auto un{ static_cast( n ) }; - for ( ;; ) { - auto & node{ this->node() }; - auto const available_offset{ static_cast( node.num_vals - 1 - value_offset_ ) }; - if ( available_offset >= un ) [[ likely ]] { - value_offset_ += static_cast( un ); - break; - } else { - un -= available_offset; - node_slot_ = node.next; - value_offset_ = 0; - BOOST_ASSERT_MSG( node_slot_ || un == 0, "Incrementing out of bounds" ); + else + { + auto un{ static_cast( -n ) }; + BOOST_ASSERT_MSG( index_ >= un, "Moving iterator out of bounds" ); + for ( ;; ) + { + auto & node{ this->node() }; + auto const available_offset{ value_offset_ }; + if ( available_offset >= un ) [[ likely ]] { + value_offset_ -= static_cast( un ); + break; + } else { + un -= available_offset; + node_slot_ = node.left; + value_offset_ = 0; + BOOST_ASSERT_MSG( node_slot_ || un == 0, "Incrementing out of bounds" ); + } } + index_ += static_cast( -n ); } - index_ += static_cast( n ); + return *this; } @@ -127,18 +216,19 @@ void bptree_base::swap( bptree_base & other ) noexcept } [[ gnu::cold ]] -bptree_base::linked_node_header & +bptree_base::node_header & bptree_base::create_root() { BOOST_ASSUME( !hdr().root_ ); BOOST_ASSUME( !hdr().depth_ ); BOOST_ASSUME( !hdr().size_ ); // the initial/'lone' root node is a leaf node - auto & root{ static_cast( static_cast( new_node() ) ) }; + auto & root{ new_node() }; auto & hdr { this->hdr() }; BOOST_ASSUME( root.num_vals == 0 ); root.num_vals = 1; - root.next = {}; + root.left = {}; + root.right = {}; hdr.root_ = slot_of( root ); hdr.leaves_ = hdr.root_; hdr.depth_ = 1; @@ -154,8 +244,16 @@ void bptree_base::free( node_header & node ) noexcept BOOST_ASSUME( node.num_vals == 0 ); auto & free_list{ hdr().free_list_ }; auto & free_node{ static_cast( node ) }; - if ( free_list ) free_node.next = free_list; - else free_node.next = {}; +#ifndef NDEBUG + auto const free_node_slot{ slot_of( free_node ) }; + if ( free_node.left ) + BOOST_ASSERT( this->node( free_node.left ).right != free_node_slot ); + if ( free_node.right ) + BOOST_ASSERT( this->node( free_node.right ).left != free_node_slot ); +#endif + free_node.left = {}; + if ( free_list ) free_node.right = free_list; + else free_node.right = {}; free_list = slot_of( free_node ); } @@ -165,6 +263,7 @@ bptree_base::slot_of( node_header const & node ) const noexcept return { static_cast( static_cast( &node ) - nodes_.data() ) }; } + [[ gnu::noinline ]] bptree_base::node_placeholder & bptree_base::new_node() @@ -173,13 +272,15 @@ bptree_base::new_node() if ( free_list ) { auto & cached_node{ node( free_list ) }; - BOOST_ASSUME( cached_node.num_vals == 0 ); - free_list = cached_node.next; - cached_node.next = {}; + BOOST_ASSUME( !cached_node.num_vals ); + BOOST_ASSUME( !cached_node.left ); + free_list = cached_node.right; + cached_node.right = {}; return as( cached_node ); } auto & new_node{ nodes_.emplace_back() }; BOOST_ASSUME( new_node.num_vals == 0 ); + new_node.left = new_node.right = {}; return new_node; } diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 96c34ed..d31a4bc 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -29,6 +29,29 @@ TEST( bp_tree, playground ) auto numbers{ std::ranges::to( sorted_numbers ) }; std::shuffle( numbers.begin(), numbers.end(), rng ); + { + bp_tree bpt; + bpt.map_memory(); + bpt.reserve( numbers.size() ); + + for ( auto const & n : numbers ) + bpt.insert( n ); + + static_assert( std::forward_iterator::const_iterator> ); + + EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ) ) ); + EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); + EXPECT_NE( bpt.find( +42 ), bpt.end() ); + EXPECT_EQ( bpt.find( -42 ), bpt.end() ); + bpt.erase( 42 ); + EXPECT_EQ( bpt.find( +42 ), bpt.end() ); + bpt.insert( 42 ); + + std::shuffle( numbers.begin(), numbers.end(), rng ); + for ( auto const & n : numbers ) + bpt.erase( n ); + } + { bp_tree bpt; bpt.map_file( test_file, flags::named_object_construction_policy::create_new_or_truncate_existing ); From 7054d84b0514727bfecbce3ddf8dae25031fc642 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Wed, 2 Oct 2024 14:30:14 +0200 Subject: [PATCH 18/49] Optimizations in the core find() function. Linux: fixed compiler warnings and linker errors. --- CMakeLists.txt | 4 ++- include/psi/vm/containers/b+tree.hpp | 39 ++++++++++++---------- include/psi/vm/containers/b+tree_print.hpp | 2 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b8bc9a..226bfce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,9 @@ if ( ${CMAKE_SYSTEM_NAME} MATCHES "Linux" ) PSI_add_link_options( Release -flto ) # lld does not seem to be enough add_compile_options( -stdlib=libc++ ) # Needed under WSL for some reason? - PSI_add_link_options( Debug -lc++ -lc++abi -lubsan ) + PSI_add_link_options( Debug -lc++ -lc++abi -lm -lubsan ) + PSI_add_link_options( DevRelease -lc++ -lc++abi -lm -lubsan ) + PSI_add_link_options( Release -lc++ -lc++abi -lm ) else() set( CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE true ) PSI_add_link_options( Release ${PSI_linker_LTO} ) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 076439d..ac721d4 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -213,13 +213,13 @@ class bptree_base template void rshift_chldrn( N & parent, auto... args ) noexcept { - auto const shifted_children{ rshift<&N::children>( parent, args... ) }; + auto const shifted_children{ rshift<&N::children>( parent, static_cast( args )... ) }; for ( auto ch_slot : shifted_children ) node( ch_slot ).parent_child_idx++; } template void lshift_chldrn( N & parent, auto... args ) noexcept { - auto const shifted_children{ lshift<&N::children>( parent, args... ) }; + auto const shifted_children{ lshift<&N::children>( parent, static_cast( args )... ) }; for ( auto ch_slot : shifted_children ) node( ch_slot ).parent_child_idx--; } @@ -353,7 +353,7 @@ class bptree_base_wkey : public bptree_base void reserve( size_type additional_values ) { additional_values = additional_values * 3 / 2; // TODO find the appropriate formula - bptree_base::reserve( ( additional_values + leaf_node::max_values - 1 ) / leaf_node::max_values ); + bptree_base::reserve( static_cast( ( additional_values + leaf_node::max_values - 1 ) / leaf_node::max_values ) ); } // solely a debugging helper (include b+tree_print.hpp) @@ -458,7 +458,7 @@ class bptree_base_wkey : public bptree_base if ( p_node->is_root() ) [[ unlikely ]] { new_root( node_slot, new_slot, std::move( key_to_propagate ) ); } else { - auto const key_pos{ p_new_node->parent_child_idx /*it is the _right_ child*/ - 1 }; + auto const key_pos{ static_cast( p_new_node->parent_child_idx /*it is the _right_ child*/ - 1 ) }; return insert( node( p_node->parent ), key_pos, std::move( key_to_propagate ), new_slot ); } } @@ -624,7 +624,7 @@ class bptree_base_wkey : public bptree_base rshift_keys( target_node, target_node_pos ); target_node.keys[ target_node_pos ] = v; if constexpr ( requires { target_node.children; } ) { - auto const ch_pos{ target_node_pos + /*>right< child*/ 1 }; + node_size_type const ch_pos( target_node_pos + /*>right< child*/ 1 ); rshift_chldrn( target_node, ch_pos ); this->insrt_child( target_node, ch_pos, right_child ); } @@ -968,7 +968,7 @@ void bptree_base_wkey::move_chldrn auto const src_chldrn{ &source.children[ src_begin ] }; auto const target_slot{ slot_of( target ) }; - for ( auto ch{ 0 }; ch < count; ++ch ) + for ( node_size_type ch{ 0 }; ch < count; ++ch ) { auto & ch_slot{ src_chldrn[ ch ] }; auto & child { node( ch_slot ) }; @@ -1079,8 +1079,8 @@ class bp_tree BOOST_NOINLINE const_iterator find( key_const_arg key ) const noexcept { auto const & leaf{ const_cast( *this ).find_nodes_for( key ).leaf }; - auto const pos{ find( leaf, key ) }; - if ( pos != leaf.num_vals && leaf.keys[ pos ] == key ) [[ likely ]] { + auto const [pos, exact_find]{ find( leaf, key ) }; + if ( exact_find ) [[ likely ]] { return iterator{ const_cast( *this ).nodes_, slot_of( leaf ), pos }; } @@ -1103,7 +1103,7 @@ class bp_tree verify( leaf ); BOOST_ASSUME( leaf.num_vals >= leaf.min_values ); } - auto const key_offset{ find( leaf, key ) }; + auto const [key_offset, key_found]{ find( leaf, key ) }; // code below can go into base if ( nodes.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" { @@ -1118,7 +1118,7 @@ class bp_tree } - BOOST_ASSUME( key_offset != leaf.num_vals && leaf.keys[ key_offset ] == key ); + BOOST_ASSUME( key_found ); lshift_keys( leaf, key_offset ); BOOST_ASSUME( leaf.num_vals ); --leaf.num_vals; @@ -1155,13 +1155,15 @@ class bp_tree [[ nodiscard ]] Comparator & mutable_comp() noexcept { return *this; } private: - [[ gnu::pure, gnu::hot, clang::preserve_most, gnu::noinline ]] - auto find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept + struct [[ clang::trivial_abi ]] find_pos { node_size_type pos; bool exact_find; }; + [[ using gnu: pure, hot, noinline, sysv_abi ]] + find_pos find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept { BOOST_ASSUME( num_vals > 0 ); - auto const pos_iter{ std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp() ) }; - auto const pos_idx { std::distance( &keys[ 0 ], pos_iter ) }; - return static_cast( pos_idx ); + auto const pos_iter { std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp() ) }; + auto const pos_idx { static_cast( std::distance( &keys[ 0 ], pos_iter ) ) }; + auto const exact_find{ ( pos_idx != num_vals ) & !comp()( value, keys[ std::min( pos_idx, num_vals ) ] ) }; + return { pos_idx, reinterpret_cast( exact_find ) }; } auto find( auto const & node, key_const_arg value ) const noexcept { @@ -1171,7 +1173,7 @@ class bp_tree template void insert( N & target_node, key_const_arg v, node_slot const right_child ) { - auto const pos{ find( target_node, v ) }; + auto const pos{ find( target_node, v ).pos }; base::insert( target_node, pos, v, right_child ); } @@ -1193,10 +1195,11 @@ class bp_tree // through the incorrectly typed reference auto p_node{ &bptree_base::as( root() ) }; auto const depth { this->hdr().depth_ }; + BOOST_ASSUME( depth >= 1 ); for ( auto level{ 0 }; level < depth - 1; ++level ) { - auto pos{ find( *p_node, key ) }; - if ( pos != p_node->num_vals && p_node->keys[ pos ] == key ) + auto [pos, exact_find]{ find( *p_node, key ) }; + if ( exact_find ) { // separator key - it also means we have to traverse to the right BOOST_ASSUME( !separator_key_node ); diff --git a/include/psi/vm/containers/b+tree_print.hpp b/include/psi/vm/containers/b+tree_print.hpp index 10f27fa..1042b21 100644 --- a/include/psi/vm/containers/b+tree_print.hpp +++ b/include/psi/vm/containers/b+tree_print.hpp @@ -20,7 +20,7 @@ void bptree_base_wkey::print() const // BFS, one level of the tree at a time. auto p_node{ &as( root() ) }; - for ( auto level{ 0U }; !is_leaf_level( level ); ++level ) + for ( depth_t level{ 0 }; !is_leaf_level( level ); ++level ) { std::print( "Level {}:\t", std::uint16_t( level ) ); From 6f31e248261344cb14383e91b0a6f9cbe09310b3 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Wed, 2 Oct 2024 21:59:25 +0200 Subject: [PATCH 19/49] Added an optimized version/fast-path of find() for cases where a linear search might be faster than binary search. Moved more shared/duplicated logic up into find_nodes_for(). --- include/psi/vm/containers/b+tree.hpp | 100 ++++++++++++++++++++------- 1 file changed, 75 insertions(+), 25 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index ac721d4..015ba8a 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include //------------------------------------------------------------------------------ namespace psi::vm @@ -38,8 +39,31 @@ namespace detail BOOST_ASSERT( remaining_space >= sizeof( Header ) ); return std::pair{ reinterpret_cast
( data ), std::span{ data + sizeof( Header ), remaining_space - sizeof( Header ) } }; } + + template static constexpr bool is_simple_comparator{ false }; + template static constexpr bool is_simple_comparator>{ true }; + template static constexpr bool is_simple_comparator>{ true }; } // namespace detail + +template +constexpr bool use_linear_search_for_sorted_array( [[ maybe_unused ]] std::uint32_t const minimum_array_length, std::uint32_t const maximum_array_length ) noexcept +{ + auto const basic_test + { + detail::is_simple_comparator && + std::is_trivially_copyable_v && + sizeof( Key ) < ( 4 * sizeof( void * ) ) && + maximum_array_length < 2048 + }; + if constexpr ( requires{ Key{}.size(); } ) + { + return basic_test && ( Key{}.size() != 0 ); + } + return basic_test; +} + + //////////////////////////////////////////////////////////////////////////////// // \class bptree_base //////////////////////////////////////////////////////////////////////////////// @@ -1071,17 +1095,20 @@ class bp_tree return; } - insert( find_nodes_for( v ).leaf, v, { /*insertion starts from leaves which do not have children*/ } ); + auto const locations{ find_nodes_for( v ) }; + BOOST_ASSUME( !locations.inner ); + BOOST_ASSUME( !locations.inner_offset ); + BOOST_ASSUME( !locations.leaf_offset.exact_find ); + base::insert( locations.leaf, locations.leaf_offset.pos, v, { /*insertion starts from leaves which do not have children*/ } ); ++this->hdr().size_; } BOOST_NOINLINE const_iterator find( key_const_arg key ) const noexcept { - auto const & leaf{ const_cast( *this ).find_nodes_for( key ).leaf }; - auto const [pos, exact_find]{ find( leaf, key ) }; - if ( exact_find ) [[ likely ]] { - return iterator{ const_cast( *this ).nodes_, slot_of( leaf ), pos }; + auto const location{ const_cast( *this ).find_nodes_for( key ) }; + if ( location.leaf_offset.exact_find ) [[ likely ]] { + return iterator{ const_cast( *this ).nodes_, slot_of( location.leaf ), location.leaf_offset.pos }; } return this->cend(); @@ -1095,31 +1122,30 @@ class bp_tree auto & __restrict root_ { hdr.root_ }; auto & __restrict size_ { hdr.size_ }; - auto const nodes{ find_nodes_for( key ) }; - - leaf_node & leaf{ nodes.leaf }; + auto const location{ find_nodes_for( key ) }; + // code below can go into base + leaf_node & leaf{ location.leaf }; if ( depth_ != 1 ) { verify( leaf ); BOOST_ASSUME( leaf.num_vals >= leaf.min_values ); } - auto const [key_offset, key_found]{ find( leaf, key ) }; - // code below can go into base - if ( nodes.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" + auto const leaf_key_offset{ location.leaf_offset.pos }; + if ( location.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" { - BOOST_ASSUME( key_offset == 0 ); - BOOST_ASSUME( leaf.keys[ key_offset ] == key ); + BOOST_ASSUME( leaf_key_offset == 0 ); + BOOST_ASSUME( leaf.keys[ leaf_key_offset ] == key ); - auto & inner { node( nodes.inner ) }; - auto & separator_key{ inner.keys[ nodes.inner_offset ] }; + auto & inner { node( location.inner ) }; + auto & separator_key{ inner.keys[ location.inner_offset ] }; BOOST_ASSUME( separator_key == key ); - BOOST_ASSUME( key_offset + 1 < leaf.num_vals ); - separator_key = leaf.keys[ key_offset + 1 ]; + BOOST_ASSUME( leaf_key_offset + 1 < leaf.num_vals ); + separator_key = leaf.keys[ leaf_key_offset + 1 ]; } - BOOST_ASSUME( key_found ); - lshift_keys( leaf, key_offset ); + BOOST_ASSUME( location.leaf_offset.exact_find ); + lshift_keys( leaf, leaf_key_offset ); BOOST_ASSUME( leaf.num_vals ); --leaf.num_vals; if ( depth_ == 1 ) [[ unlikely ]] // handle 'leaf root' deletion directly to simplify handle_underflow() @@ -1155,14 +1181,30 @@ class bp_tree [[ nodiscard ]] Comparator & mutable_comp() noexcept { return *this; } private: - struct [[ clang::trivial_abi ]] find_pos { node_size_type pos; bool exact_find; }; + struct find_pos + { + node_size_type pos : ( sizeof( node_size_type ) * CHAR_BIT - 1 ); + node_size_type exact_find : 1; + }; [[ using gnu: pure, hot, noinline, sysv_abi ]] find_pos find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept { BOOST_ASSUME( num_vals > 0 ); - auto const pos_iter { std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp() ) }; - auto const pos_idx { static_cast( std::distance( &keys[ 0 ], pos_iter ) ) }; - auto const exact_find{ ( pos_idx != num_vals ) & !comp()( value, keys[ std::min( pos_idx, num_vals ) ] ) }; + auto const & __restrict comp{ this->comp() }; + node_size_type pos_idx; + if constexpr ( use_linear_search_for_sorted_array( 1, leaf_node::max_values ) ) + { + auto k{ 0 }; + while ( ( k != num_vals ) && comp( keys[ k ], value ) ) + ++k; + pos_idx = static_cast( k ); + } + else + { + auto const pos_iter{ std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp ) }; + pos_idx = static_cast( std::distance( &keys[ 0 ], pos_iter ) ); + } + auto const exact_find{ ( pos_idx != num_vals ) & !comp( value, keys[ std::min( pos_idx, num_vals - 1 ) ] ) }; return { pos_idx, reinterpret_cast( exact_find ) }; } auto find( auto const & node, key_const_arg value ) const noexcept @@ -1180,9 +1222,10 @@ class bp_tree struct key_locations { leaf_node & leaf; + find_pos leaf_offset; // optional - if also present in an inner node as a separator key + node_size_type inner_offset; // ordered for compact layout node_slot inner; - node_size_type inner_offset; }; [[ gnu::hot, gnu::sysv_abi ]] @@ -1209,7 +1252,14 @@ class bp_tree } p_node = &node( p_node->children[ pos ] ); } - return { base::template as( *p_node ), separator_key_node, separator_key_offset }; + auto & leaf{ base::template as( *p_node ) }; + return + { + leaf, + separator_key_node ? find_pos{ 0, true } : find( leaf, key ), + separator_key_offset, + separator_key_node + }; } }; // class bp_tree From fd5b9e3907fdcc813c1434ebc855559a7d334a91 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Wed, 2 Oct 2024 22:45:31 +0200 Subject: [PATCH 20/49] Fixed cases of "direct" key comparison (ignoring/bypassing the user specified comparator). --- include/psi/vm/containers/b+tree.hpp | 36 +++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 015ba8a..c196a49 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -739,15 +739,16 @@ class bptree_base_wkey : public bptree_base if constexpr ( leaf_node_type ) { // Move the smallest key from the right sibling to the current node auto & leftmost_right_key{ keys( *p_right_sibling ).front() }; - BOOST_ASSUME( *p_right_separator_key == leftmost_right_key ); + BOOST_ASSUME( *p_right_separator_key == leftmost_right_key ); // yes we expect exact or bitwise equality for key-copies in inner nodes node_keys.back() = std::move( leftmost_right_key ); lshift_keys( *p_right_sibling ); // adjust the separator key in the parent *p_right_separator_key = leftmost_right_key; } else { // Move/rotate the smallest key from the right sibling to the current node 'through' the parent - BOOST_ASSUME( parent.keys[ parent_child_idx ] < p_right_sibling->keys[ 0 ] ); - BOOST_ASSERT( *( node_keys.end() - 2 ) < *p_right_separator_key ); + // no comparator in base classes :/ + //BOOST_ASSUME( le( parent.keys[ parent_child_idx ], p_right_sibling->keys[ 0 ] ) ); + //BOOST_ASSERT( le( *( node_keys.end() - 2 ), *p_right_separator_key ) ); node_keys.back() = std::move( *p_right_separator_key ); *p_right_separator_key = std::move( keys( *p_right_sibling ).front() ); insrt_child( node, num_chldrn( node ) - 1, children( *p_right_sibling ).front(), node_slot ); @@ -768,13 +769,14 @@ class bptree_base_wkey : public bptree_base verify( *p_left_sibling ); BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); // Merge node -> left sibling - this->merge_right_into_left( *p_left_sibling, node, parent, left_separator_key_idx, parent_child_idx ); + merge_right_into_left( *p_left_sibling, node, parent, left_separator_key_idx, parent_child_idx ); } else { verify( *p_right_sibling ); BOOST_ASSUME( parent_key_idx == 0 ); - BOOST_ASSUME( parent.keys[ parent_key_idx ] <= p_right_sibling->keys[ 0 ] ); + // no comparator in base classes :/ + //BOOST_ASSUME( leq( parent.keys[ parent_key_idx ], p_right_sibling->keys[ 0 ] ) ); // Merge right sibling -> node - this->merge_right_into_left( node, *p_right_sibling, parent, right_separator_key_idx, static_cast( parent_child_idx + 1 ) ); + merge_right_into_left( node, *p_right_sibling, parent, right_separator_key_idx, static_cast( parent_child_idx + 1 ) ); } // propagate underflow @@ -1134,11 +1136,11 @@ class bp_tree if ( location.inner ) [[ unlikely ]] // "most keys are in the leaf nodes" { BOOST_ASSUME( leaf_key_offset == 0 ); - BOOST_ASSUME( leaf.keys[ leaf_key_offset ] == key ); + BOOST_ASSUME( eq( leaf.keys[ leaf_key_offset ], key ) ); auto & inner { node( location.inner ) }; auto & separator_key{ inner.keys[ location.inner_offset ] }; - BOOST_ASSUME( separator_key == key ); + BOOST_ASSUME( eq( separator_key, key ) ); BOOST_ASSUME( leaf_key_offset + 1 < leaf.num_vals ); separator_key = leaf.keys[ leaf_key_offset + 1 ]; } @@ -1261,6 +1263,24 @@ class bp_tree separator_key_node }; } + + [[ gnu::pure ]] bool le( key_const_arg left, key_const_arg right ) const noexcept { return comp()( left, right ); } + [[ gnu::pure ]] bool eq( key_const_arg left, key_const_arg right ) const noexcept + { + if constexpr ( requires{ comp().eq( left, right ); } ) + return comp().eq( left, right ); + if constexpr ( detail::is_simple_comparator && requires{ left == right; } ) + return left == right; + return !comp()( left, right ) && !comp()( right, left ); + } + [[ gnu::pure ]] bool leq( key_const_arg left, key_const_arg right ) const noexcept + { + if constexpr ( requires{ comp().leq( left, right ); } ) + return comp().leq( left, right ); + if constexpr ( detail::is_simple_comparator && requires { left == right; } ) + return left <= right; + return !comp()( right, left ); + } }; // class bp_tree PSI_WARNING_DISABLE_POP() From 1c1d54737679a319dd3f2d8c8272502922af0e06 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Fri, 11 Oct 2024 13:17:58 +0200 Subject: [PATCH 21/49] Windows: decreased (resizable) anonymous mappings maximum size to 2GB (quick-fix for sporadic failures on an 8GB Win11 machine). --- src/mappable_objects/file/file.win32.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mappable_objects/file/file.win32.cpp b/src/mappable_objects/file/file.win32.cpp index 133ea1c..605e6aa 100644 --- a/src/mappable_objects/file/file.win32.cpp +++ b/src/mappable_objects/file/file.win32.cpp @@ -121,10 +121,10 @@ namespace detail if ( !file ) { // Windows (11 23H2) does not (still) seem to support resizing of pagefile-backed mappings - // so we emulate those by creating a 4GB one and counting on: + // so we emulate those by creating a 2GB one and counting on: // - the NT kernel to be lazy and // - that amount to be 'enough for everyone'. - maximum_size.QuadPart = std::numeric_limits::max(); + maximum_size.QuadPart = std::numeric_limits::max(); } auto const nt_result From 0fda44055b825cf7306c1db006be27d3faa10c76 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Fri, 11 Oct 2024 13:18:21 +0200 Subject: [PATCH 22/49] Minor cleanup. --- include/psi/vm/flags/mapping.win32.hpp | 2 +- src/flags/mapping.win32.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/psi/vm/flags/mapping.win32.hpp b/include/psi/vm/flags/mapping.win32.hpp index cedec9b..88da836 100644 --- a/include/psi/vm/flags/mapping.win32.hpp +++ b/include/psi/vm/flags/mapping.win32.hpp @@ -72,7 +72,7 @@ struct [[ clang::trivial_abi ]] viewing namespace detail { - flags_t BOOST_CC_REG object_access_to_page_access( access_privileges::object, viewing::share_mode ); + flags_t object_access_to_page_access( access_privileges::object, viewing::share_mode ) noexcept; } // namespace detail diff --git a/src/flags/mapping.win32.cpp b/src/flags/mapping.win32.cpp index d8e9951..6c42e40 100644 --- a/src/flags/mapping.win32.cpp +++ b/src/flags/mapping.win32.cpp @@ -68,7 +68,7 @@ bool viewing::is_cow() const noexcept namespace detail { - flags_t BOOST_CC_REG object_access_to_page_access( access_privileges::object const object_access, viewing::share_mode const share_mode ) + flags_t object_access_to_page_access( access_privileges::object const object_access, viewing::share_mode const share_mode ) noexcept { // Generate CreateFileMapping flags from access_privileges::object/MapViewOfFile flags using access_rights = access_privileges; From 60525d575d2d2607d338d95898d5ff20bb88d7be Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Fri, 11 Oct 2024 13:19:26 +0200 Subject: [PATCH 23/49] Replaced usage of std::tuple as a size-holder with a custom type (more readable while debugging). --- include/psi/vm/vector.hpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index e11e11e..8efaa5c 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -43,6 +43,9 @@ namespace vm namespace detail { [[ noreturn ]] inline void throw_out_of_range() { throw std::out_of_range( "vm::vector access out of bounds" ); } + + template struct size { T value; }; + template struct size {}; } // namespace detail class contiguous_container_storage_base @@ -185,7 +188,7 @@ template class contiguous_container_storage : public contiguous_container_storage_base, - private std::conditional_t> + private detail::size // checkout a revision prior to March the 21st 2024 for a version that used statically sized header sizes // this approach is more versatile while the overhead should be less than negligible { @@ -194,16 +197,17 @@ class contiguous_container_storage private: static constexpr auto size_size{ headerless ? 0 : sizeof( size_type ) }; + using size_holder = detail::size; public: contiguous_container_storage( ) noexcept requires( headerless ) = default; - explicit contiguous_container_storage( size_type const header_size ) noexcept requires( !headerless ) : std::tuple{ header_size } {} + explicit contiguous_container_storage( size_type const header_size ) noexcept requires( !headerless ) : size_holder{ header_size } {} static constexpr std::uint8_t header_size() noexcept requires( headerless ) { return 0; } [[ gnu::pure ]] size_type header_size() const noexcept requires( !headerless ) { - auto const sz{ std::get<0>( *this ) }; + auto const sz{ size_holder::value }; BOOST_ASSUME( sz >= sizeof( sz_t ) ); return sz; } From c3e96289810389e1f0e381e5e62495d310dccb1f Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Fri, 11 Oct 2024 14:51:11 +0200 Subject: [PATCH 24/49] Added an efficient bulk insertion method. Improved the iterators implementation: - support decrementable end iterators - optimized decrement operators. Fixed public bptree::find() not to crash on empty bptrees. Fixed bptree_base::reserve() to take into account existing capacity/free nodes. Expanded the unit test. Minor other related cleanups. --- include/psi/vm/containers/b+tree.hpp | 522 ++++++++++++++++++++++----- src/containers/b+tree.cpp | 168 ++++++--- test/b+tree.cpp | 32 +- 3 files changed, 575 insertions(+), 147 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index c196a49..8117bec 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -75,7 +75,7 @@ class bptree_base using difference_type = std::make_signed_t; using storage_result = err::fallible_result; - [[ gnu::pure ]] auto empty() const noexcept { return size() == 0; } + [[ gnu::pure ]] bool empty() const noexcept { return BOOST_UNLIKELY( size() == 0 ); } void clear() noexcept; @@ -102,7 +102,7 @@ class bptree_base using value_type = std::uint32_t; static node_slot const null; value_type index{ static_cast( -1 ) }; // in-pool index/offset - [[ gnu::pure ]] value_type operator*() const noexcept { return index; } + [[ gnu::pure ]] value_type operator*() const noexcept { BOOST_ASSUME( index != null.index ); return index; } [[ gnu::pure ]] bool operator==( node_slot const other ) const noexcept { return this->index == other.index; } [[ gnu::pure ]] explicit operator bool() const noexcept { return index != null.index; } }; // struct node_slot @@ -142,16 +142,23 @@ class bptree_base struct alignas( node_size ) free_node : node_header {}; // SCARY iterator parts + struct iter_pos + { + node_slot node {}; + node_size_type value_offset{}; + }; class base_iterator; class base_random_access_iterator; struct header { - node_slot root_; - node_slot free_list_; - node_slot leaves_; - size_t size_; - depth_t depth_; + node_slot root_; + node_slot first_leaf_; + node_slot last_leaf_; + node_slot free_list_; + node_slot::value_type free_node_count_{}; + size_t size_ {}; + depth_t depth_{}; }; // struct header using node_pool = vm::vector; @@ -159,6 +166,15 @@ class bptree_base protected: void swap( bptree_base & other ) noexcept; + [[ gnu::pure ]] iter_pos begin_pos() const noexcept; + [[ gnu::pure ]] iter_pos end_pos() const noexcept; + + [[ gnu::pure ]] base_iterator begin() noexcept; + [[ gnu::pure ]] base_iterator end () noexcept; + + [[ gnu::pure ]] base_random_access_iterator ra_begin() noexcept; + [[ gnu::pure ]] base_random_access_iterator ra_end () noexcept; + [[ gnu::pure ]] size_type size() const noexcept { return hdr().size_; } [[ gnu::cold ]] node_header & create_root(); @@ -176,7 +192,7 @@ class bptree_base [[ gnu::pure ]] header & hdr() noexcept; [[ gnu::pure ]] header const & hdr() const noexcept { return const_cast( *this ).hdr(); } - node_slot first_leaf() const noexcept { return hdr().leaves_; } + node_slot first_leaf() const noexcept { return hdr().first_leaf_; } static void verify( auto const & node ) noexcept { @@ -294,6 +310,9 @@ class bptree_base protected: node_pool nodes_; +#ifndef NDEBUG + header const * hdr_{}; +#endif }; // class bptree_base inline constexpr bptree_base::node_slot const bptree_base::node_slot::null{ static_cast( -1 ) }; @@ -306,27 +325,34 @@ inline constexpr bptree_base::node_slot const bptree_base::node_slot::null{ stat class bptree_base::base_iterator { protected: + mutable #ifndef NDEBUG // for bounds checking std::span #else node_placeholder * __restrict #endif - nodes_ {}; - node_slot node_slot_ {}; - node_size_type value_offset_{}; + nodes_{}; + iter_pos pos_ {}; + friend class bptree_base; template friend class bp_tree; - base_iterator( node_pool &, node_slot, node_size_type value_offset ) noexcept; + base_iterator( node_pool &, iter_pos ) noexcept; [[ gnu::pure ]] node_header & node() const noexcept; + void update_pool_ptr( node_pool & ) const noexcept; + public: constexpr base_iterator() noexcept = default; base_iterator & operator++() noexcept; + base_iterator & operator--() noexcept; bool operator==( base_iterator const & ) const noexcept; + +public: + auto pos() const noexcept { return pos_; } }; // class base_iterator //////////////////////////////////////////////////////////////////////////////// @@ -338,10 +364,11 @@ class bptree_base::base_random_access_iterator : public base_iterator protected: size_type index_; + friend class bptree_base; template friend class bp_tree; - base_random_access_iterator( bptree_base & parent, node_slot const start_leaf, size_type const start_index ) noexcept - : base_iterator{ parent.nodes_, start_leaf, 0 }, index_{ start_index } {} + base_random_access_iterator( bptree_base & parent, iter_pos const pos, size_type const start_index ) noexcept + : base_iterator{ parent.nodes_, pos }, index_{ start_index } {} public: constexpr base_random_access_iterator() noexcept = default; @@ -352,6 +379,8 @@ class bptree_base::base_random_access_iterator : public base_iterator base_random_access_iterator & operator++( ) noexcept { base_iterator::operator++(); ++index_; return *this; } base_random_access_iterator operator++(int) noexcept { auto current{ *this }; operator++(); return current; } + base_random_access_iterator & operator--( ) noexcept { base_iterator::operator--(); --index_; return *this; } + base_random_access_iterator operator--(int) noexcept { auto current{ *this }; operator--(); return current; } // should implicitly handle end iterator comparison also (this requires the start_index constructor argument for the construction of end iterators) friend constexpr bool operator==( base_random_access_iterator const & left, base_random_access_iterator const & right ) noexcept { return left.index_ == right.index_; } @@ -386,7 +415,7 @@ class bptree_base_wkey : public bptree_base protected: // node types struct alignas( node_size ) parent_node : node_header { - static auto constexpr storage_space{ node_size - psi::vm::align_up( sizeof( node_header ), alignof( Key ) ) }; + static auto constexpr storage_space{ node_size - align_up( sizeof( node_header ), alignof( Key ) ) }; // storage_space = ( order - 1 ) * sizeof( key ) + order * sizeof( child_ptr ) // storage_space = order * szK - szK + order * szC @@ -427,7 +456,7 @@ class bptree_base_wkey : public bptree_base // TODO support for maps (i.e. keys+values) using value_type = Key; - static node_size_type constexpr storage_space{ node_size - psi::vm::align_up( sizeof( node_header ), alignof( Key ) ) }; + static node_size_type constexpr storage_space{ node_size - align_up( sizeof( node_header ), alignof( Key ) ) }; static node_size_type constexpr max_values { storage_space / sizeof( Key ) }; static node_size_type constexpr min_values { ihalf_ceil }; @@ -452,40 +481,7 @@ class bptree_base_wkey : public bptree_base class ra_iterator; protected: // split_to_insert and its helpers - template - void split_to_insert( N & node_to_split, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) - { - auto [ node_slot, new_slot ]{ bptree_base::new_spillover_node_for( node_to_split ) }; - auto const max{ N::max_values }; - auto const mid{ N::min_values }; - auto p_node { &node( node_slot ) }; - auto p_new_node{ &node( new_slot ) }; - verify( *p_node ); - BOOST_ASSUME( p_node->num_vals == max ); - BOOST_ASSERT - ( - !p_node->parent || ( node( p_node->parent ).children[ p_node->parent_child_idx ] == node_slot ) - ); - - auto const new_insert_pos { insert_pos - mid }; - bool const insertion_into_new_node{ new_insert_pos >= 0 }; - auto key_to_propagate{ insertion_into_new_node // we cannot save a reference here because it might get invalidated by the new_node() call below - ? insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) - : insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) - }; - - verify( *p_node ); - verify( *p_new_node ); - BOOST_ASSUME( p_node->num_vals == mid ); - - // propagate the mid key to the parent - if ( p_node->is_root() ) [[ unlikely ]] { - new_root( node_slot, new_slot, std::move( key_to_propagate ) ); - } else { - auto const key_pos{ static_cast( p_new_node->parent_child_idx /*it is the _right_ child*/ - 1 ) }; - return insert( node( p_node->parent ), key_pos, std::move( key_to_propagate ), new_slot ); - } - } + struct insert_pos_t { node_slot node; node_size_type next_insert_offset; }; void new_root( node_slot const left_child, node_slot const right_child, key_const_arg separator_key ) { @@ -495,13 +491,14 @@ class bptree_base_wkey : public bptree_base new_root_node.children[ 1 ] = right_child; } - value_type insert_into_new_node + auto insert_into_new_node ( inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_size_type const new_insert_pos, node_slot const key_right_child - ) noexcept { + ) noexcept + { BOOST_ASSUME( bool( key_right_child ) ); using N = inner_node; @@ -542,16 +539,17 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( !underflowed( node ) ); BOOST_ASSUME( !underflowed( new_node ) ); - return key_to_propagate; + return std::make_pair( key_to_propagate, new_insert_pos ); } - static value_type const & insert_into_new_node + static auto insert_into_new_node ( leaf_node & node, leaf_node & new_node, key_const_arg value, node_size_type const insert_pos, node_size_type const new_insert_pos, node_slot const key_right_child - ) noexcept { + ) noexcept + { BOOST_ASSUME( !key_right_child ); using N = leaf_node; @@ -574,10 +572,10 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( !underflowed( node ) ); BOOST_ASSUME( !underflowed( new_node ) ); - return key_to_propagate; + return std::make_pair( key_to_propagate, static_cast( new_insert_pos + 1 ) ); } - value_type insert_into_existing_node( inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + auto insert_into_existing_node( inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept { BOOST_ASSUME( bool( key_right_child ) ); @@ -606,10 +604,10 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( !underflowed( node ) ); BOOST_ASSUME( !underflowed( new_node ) ); - return key_to_propagate; + return std::make_pair( key_to_propagate, static_cast( insert_pos + 1 ) ); } - static value_type const & insert_into_existing_node( leaf_node & node, leaf_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + static auto insert_into_existing_node( leaf_node & node, leaf_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept { BOOST_ASSUME( !key_right_child ); @@ -633,12 +631,56 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( !underflowed( node ) ); BOOST_ASSUME( !underflowed( new_node ) ); - return key_to_propagate; + return std::make_pair( key_to_propagate, static_cast( insert_pos + 1 ) ); } + template + insert_pos_t split_to_insert( N & node_to_split, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) + { + auto const max{ N::max_values }; + auto const mid{ N::min_values }; + BOOST_ASSUME( node_to_split.num_vals == max ); + auto [node_slot, new_slot]{ bptree_base::new_spillover_node_for( node_to_split ) }; + auto p_node { &node( node_slot ) }; + auto p_new_node{ &node( new_slot ) }; + verify( *p_node ); + BOOST_ASSUME( p_node->num_vals == max ); + BOOST_ASSERT + ( + !p_node->parent || ( node( p_node->parent ).children[ p_node->parent_child_idx ] == node_slot ) + ); + + auto const new_insert_pos { insert_pos - mid }; + bool const insertion_into_new_node{ new_insert_pos >= 0 }; + auto [key_to_propagate, next_insert_pos]{ insertion_into_new_node // we cannot save a reference here because it might get invalidated by the new_node() call below + ? insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) + : insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) + }; + + verify( *p_node ); + verify( *p_new_node ); + BOOST_ASSUME( p_node->num_vals == mid ); + + if ( std::is_same_v && !p_new_node->right ) { + hdr().last_leaf_ = new_slot; + } + + // propagate the mid key to the parent + if ( p_node->is_root() ) [[ unlikely ]] { + new_root( node_slot, new_slot, std::move( key_to_propagate ) ); + } else { + auto const key_pos{ static_cast( p_new_node->parent_child_idx /*it is the _right_ child*/ - 1 ) }; + insert( node( p_node->parent ), key_pos, std::move( key_to_propagate ), new_slot ); + } + return insertion_into_new_node + ? insert_pos_t{ new_slot, next_insert_pos } + : insert_pos_t{ node_slot, next_insert_pos }; + } + + protected: // 'other' template - void insert( N & target_node, node_size_type const target_node_pos, key_const_arg v, node_slot const right_child ) + insert_pos_t insert( N & target_node, node_size_type const target_node_pos, key_const_arg v, node_slot const right_child ) { verify( target_node ); if ( full( target_node ) ) [[ unlikely ]] { @@ -652,7 +694,114 @@ class bptree_base_wkey : public bptree_base rshift_chldrn( target_node, ch_pos ); this->insrt_child( target_node, ch_pos, right_child ); } + return { slot_of( target_node ), static_cast( target_node_pos + 1 ) }; + } + } + + void bulk_append( leaf_node const * src_leaf, insert_pos_t rightmost_parent_pos ) noexcept + { + for ( ;; ) + { + auto & rightmost_parent{ inner( rightmost_parent_pos.node ) }; + BOOST_ASSUME( rightmost_parent_pos.next_insert_offset == rightmost_parent.num_vals ); + rightmost_parent_pos = insert + ( + rightmost_parent, + rightmost_parent_pos.next_insert_offset, + src_leaf->keys[ 0 ], + slot_of( *src_leaf ) + ); + if ( !src_leaf->right ) + break; + src_leaf = &leaf( src_leaf->right ); + } + hdr().last_leaf_ = slot_of( *src_leaf ); + } + + auto bulk_insert_prepare( std::span keys ) + { + reserve( static_cast( keys.size() ) ); + + auto & hdr{ this->hdr() }; + auto const begin { hdr.free_list_ }; + auto leaf_slot{ begin }; + while ( !keys.empty() ) + { + leaf_node & leaf{ this->leaf( leaf_slot ) }; + BOOST_ASSUME( leaf.num_vals == 0 ); + auto const size_to_copy{ static_cast( std::min( leaf.max_values, keys.size() ) ) }; + BOOST_ASSUME( size_to_copy ); + std::copy_n( keys.begin(), size_to_copy, leaf.keys ); + leaf.num_vals = size_to_copy; + keys = keys.subspan( size_to_copy ); + --hdr.free_node_count_; + if ( !keys.empty() ) + leaf_slot = leaf.right; + else + { + hdr.free_list_ = leaf.right; + node( leaf.right ).left = {}; + leaf.right = {}; + return std::make_pair( begin, iter_pos{ leaf_slot, size_to_copy } ); + } } + std::unreachable(); + } + + void bulk_insert_into_empty( node_slot const begin_leaf, iter_pos const end_leaf, size_type const total_size ) + { + BOOST_ASSUME( empty() ); + auto & hdr{ this->hdr() }; + hdr.root_ = begin_leaf; + hdr.first_leaf_ = begin_leaf; + if ( begin_leaf == end_leaf.node ) [[ unlikely ]] // single-node-sized initial insert + { + hdr.last_leaf_ = end_leaf.node; + return; + } + auto const & first_root_left { leaf( begin_leaf ) }; + auto & first_root_right{ leaf( first_root_left.right ) }; + first_root_right.parent_child_idx = 1; + hdr.depth_ = 1; + new_root( begin_leaf, first_root_left.right, first_root_right.keys[ 0 ] ); // may invalidate the hdr reference + BOOST_ASSUME( this->hdr().depth_ == 2 ); + bulk_append( &leaf( first_root_right.right ), { hdr.root_, 1 } ); + BOOST_ASSUME( this->hdr().last_leaf_ == end_leaf.node ); + this->hdr().size_ = total_size; + } + + leaf_node & leaf ( node_slot const slot ) noexcept { return node< leaf_node>( slot ); } + inner_node & inner ( node_slot const slot ) noexcept { return node( slot ); } + inner_node & parent( node_header & child ) noexcept { return inner( child.parent ); } + + void update_separator( leaf_node & leaf, Key const & new_separator ) noexcept + { + // the leftmost leaf does not have a separator key (at all) + if ( !leaf.left ) [[ unlikely ]] + { + BOOST_ASSUME( leaf.parent_child_idx == 0 ); + BOOST_ASSUME( hdr().first_leaf_ == slot_of( leaf ) ); + return; + } + auto const & separator_key{ leaf.keys[ 0 ] }; + BOOST_ASSUME( separator_key != new_separator ); + // a leftmost child does not have a key (in the immediate parent) + // (because a left child is strictly less-than its separator key - for + // the leftmost child there is no key further left that could be + // greater-or-equal to it) + auto parent_child_idx{ leaf.parent_child_idx }; + auto * parent { &this->parent( leaf ) }; + while ( parent_child_idx == 0 ) + { + parent = &this->parent( *parent ); + parent_child_idx = parent->parent_child_idx; + } + // can be zero only for the leftmost leaf which was checked for in the + // loop above and at the beginning of the function + BOOST_ASSUME( parent_child_idx > 0 ); + auto & parent_key{ parent->keys[ leaf.parent_child_idx - 1 ] }; + BOOST_ASSUME( parent_key == separator_key ); + parent_key = new_separator; } template @@ -678,17 +827,17 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( parent.children[ parent_child_idx ] == node_slot ); // the left and right level dlink pointers can point 'across' parents - // (and so cannot be used to resolve siblings) - auto const right_separator_key_idx{ static_cast( parent_key_idx + parent_has_key_copy ) }; - auto const left_separator_key_idx{ std::min( static_cast( right_separator_key_idx - 1 ), parent.num_vals ) }; // (ab)use unsigned wraparound - + // (and so cannot be used to resolve existence of siblings) auto const has_right_sibling{ parent_child_idx < ( num_chldrn( parent ) - 1 ) }; auto const has_left_sibling { parent_child_idx > 0 }; - auto const p_right_sibling{ has_right_sibling ? &this->node( node.right ) : nullptr }; - auto const p_left_sibling { has_left_sibling ? &this->node( node.left ) : nullptr }; + auto const p_right_sibling { has_right_sibling ? &this->node( node.right ) : nullptr }; + auto const p_left_sibling { has_left_sibling ? &this->node( node.left ) : nullptr }; + + auto const right_separator_key_idx{ static_cast( parent_key_idx + parent_has_key_copy ) }; + auto const left_separator_key_idx{ std::min( static_cast( right_separator_key_idx - 1 ), parent.num_vals ) }; // (ab)use unsigned wraparound + auto const p_right_separator_key { has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; + auto const p_left_separator_key { has_left_sibling ? &parent.keys[ left_separator_key_idx ] : nullptr }; - auto const p_right_separator_key{ has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; - auto const p_left_separator_key { has_left_sibling ? &parent.keys[ left_separator_key_idx ] : nullptr }; BOOST_ASSUME( has_right_sibling || has_left_sibling ); BOOST_ASSERT( &node != p_left_sibling ); BOOST_ASSERT( &node != p_right_sibling ); @@ -812,7 +961,7 @@ class bptree_base_wkey : public bptree_base using bptree_base::free; void free( leaf_node & leaf ) noexcept { - auto & first_leaf{ hdr().leaves_ }; + auto & first_leaf{ hdr().first_leaf_ }; if ( first_leaf == slot_of( leaf ) ) { BOOST_ASSUME( !leaf.left ); first_leaf = leaf.right; @@ -931,10 +1080,16 @@ class bptree_base_wkey::fwd_iterator public: constexpr fwd_iterator() noexcept = default; - Key & operator*() const noexcept { return static_cast( node() ).keys[ value_offset_ ]; } + Key & operator*() const noexcept + { + auto & leaf{ static_cast( node() ) }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + return leaf.keys[ pos_.value_offset ]; } constexpr fwd_iterator & operator++() noexcept { return static_cast( base_iterator::operator++() ); } + constexpr fwd_iterator & operator--() noexcept { return static_cast( base_iterator::operator--() ); } using impl::operator++; + using impl::operator--; }; // class fwd_iterator //////////////////////////////////////////////////////////////////////////////// @@ -947,18 +1102,27 @@ class bptree_base_wkey::ra_iterator public base_random_access_iterator, public iter_impl { -private: +private: friend class bptree_base_wkey; using base_random_access_iterator::base_random_access_iterator; + leaf_node & node() const noexcept { return static_cast( base_random_access_iterator::node() ); } + public: constexpr ra_iterator() noexcept = default; - Key & operator*() const noexcept { return static_cast( node() ).keys[ value_offset_ ]; } + Key & operator*() const noexcept + { + auto & leaf{ node() }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + return leaf.keys[ pos_.value_offset ]; + } ra_iterator & operator+=( difference_type const n ) noexcept { return static_cast( base_random_access_iterator::operator+=( n ) ); } ra_iterator & operator++( ) noexcept { return static_cast( base_random_access_iterator::operator++( ) ); } ra_iterator operator++(int) noexcept { return static_cast( base_random_access_iterator::operator++(0) ); } + ra_iterator & operator--( ) noexcept { return static_cast( base_random_access_iterator::operator--( ) ); } + ra_iterator operator--(int) noexcept { return static_cast( base_random_access_iterator::operator--(0) ); } friend constexpr bool operator==( ra_iterator const & left, ra_iterator const & right ) noexcept { return left.index_ == right.index_; } @@ -966,7 +1130,7 @@ class bptree_base_wkey::ra_iterator }; // class ra_iterator template -template [[ gnu::noinline, gnu::sysv_abi ]] +template [[ gnu::sysv_abi ]] void bptree_base_wkey::move_keys ( N const & source, node_size_type const src_begin, node_size_type const src_end, @@ -994,13 +1158,13 @@ void bptree_base_wkey::move_chldrn auto const src_chldrn{ &source.children[ src_begin ] }; auto const target_slot{ slot_of( target ) }; - for ( node_size_type ch{ 0 }; ch < count; ++ch ) + for ( node_size_type ch_idx{ 0 }; ch_idx < count; ++ch_idx ) { - auto & ch_slot{ src_chldrn[ ch ] }; + auto & ch_slot{ src_chldrn[ ch_idx ] }; auto & child { node( ch_slot ) }; - target.children[ tgt_begin + ch ] = std::move( ch_slot ); - child.parent = target_slot; - child.parent_child_idx = tgt_begin + ch; + target.children[ tgt_begin + ch_idx ] = std::move( ch_slot ); + child.parent = target_slot; + child.parent_child_idx = tgt_begin + ch_idx; } } @@ -1053,6 +1217,8 @@ class bp_tree using base::root; public: + static constexpr auto unique{ true }; // TODO non unique + using size_type = base::size_type; using value_type = base::value_type; using pointer = value_type *; @@ -1062,6 +1228,7 @@ class bp_tree using iterator = fwd_iterator; using const_iterator = std::basic_const_iterator; + using base::empty; using base::size; using base::clear; @@ -1075,11 +1242,11 @@ class bp_tree return max_sz; } - [[ gnu::pure ]] iterator begin() noexcept { return { this->nodes_, this->first_leaf(), 0 }; } using stl_impl::begin; - [[ gnu::pure ]] iterator end () noexcept { return { this->nodes_, {} , 0 }; } using stl_impl::end ; + [[ gnu::pure ]] iterator begin() noexcept { return static_cast( base::begin() ); } using stl_impl::begin; + [[ gnu::pure ]] iterator end() noexcept { return static_cast( base:: end() ); } using stl_impl::end ; - [[ gnu::pure ]] ra_iterator ra_begin() noexcept { return { *this, this->first_leaf(), 0 }; } - [[ gnu::pure ]] ra_iterator ra_end () noexcept { return { *this, {} , size() }; } + [[ gnu::pure ]] ra_iterator ra_begin() noexcept { return static_cast( base::ra_begin() ); } + [[ gnu::pure ]] ra_iterator ra_end () noexcept { return static_cast( base::ra_end () ); } [[ gnu::pure ]] std::basic_const_iterator ra_begin() const noexcept { return const_cast( *this ).ra_begin(); } [[ gnu::pure ]] std::basic_const_iterator ra_end () const noexcept { return const_cast( *this ).ra_end (); } @@ -1089,7 +1256,7 @@ class bp_tree BOOST_NOINLINE void insert( key_const_arg v ) { - if ( base::empty() ) + if ( empty() ) { auto & root{ static_cast( base::create_root() ) }; BOOST_ASSUME( root.num_vals == 1 ); @@ -1106,11 +1273,74 @@ class bp_tree ++this->hdr().size_; } - BOOST_NOINLINE - const_iterator find( key_const_arg key ) const noexcept { - auto const location{ const_cast( *this ).find_nodes_for( key ) }; - if ( location.leaf_offset.exact_find ) [[ likely ]] { - return iterator{ const_cast( *this ).nodes_, slot_of( location.leaf ), location.leaf_offset.pos }; + // bulk insert + // TODO proper std insert interface (w/ ranges, iterators, hints...) + // https://www.sciencedirect.com/science/article/abs/pii/S0020025502002025 On batch-constructing B+-trees: algorithm and its performance + // evaluation https://www.vldb.org/conf/2001/P461.pdf An Evaluation of Generic Bulk Loading Techniques + // https://stackoverflow.com/questions/15996319/is-there-any-algorithm-for-bulk-loading-in-b-tree + void insert( std::span keys ) + { + auto const total_size{ static_cast( keys.size() ) }; + auto const [begin_leaf, end_pos]{ base::bulk_insert_prepare( keys ) }; + ra_iterator const p_new_nodes_begin{ *this, { begin_leaf, 0 }, 0 }; + ra_iterator const p_new_nodes_end { *this, end_pos , total_size }; + std::sort( p_new_nodes_begin, p_new_nodes_end, comp() ); + + if ( empty() ) + return base::bulk_insert_into_empty( begin_leaf, end_pos, total_size ); + + for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) + { + auto [ source_slot, source_slot_offset ]{ p_new_keys.pos() }; + auto const tgt_location{ find_nodes_for( *p_new_keys ) }; + auto tgt_leaf { &tgt_location.leaf }; + auto tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; + + auto * src_leaf{ &base::leaf( source_slot ) }; + // if we have reached the end of the rightmost leaf simply perform + // a bulk_append + if ( ( tgt_leaf_next_pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) + { + lshift_keys( *src_leaf, source_slot_offset ); + src_leaf->num_vals -= source_slot_offset; + src_leaf->left = slot_of( *tgt_leaf ); + tgt_leaf->right = source_slot; + auto const rightmost_parent_slot{ tgt_leaf->parent }; + auto const parent_pos{ tgt_leaf->parent_child_idx }; // key idx = child idx - 1 & this is the 'next' key + base::bulk_append( src_leaf, { rightmost_parent_slot, parent_pos } ); + break; + } + + node_size_type consumed_source; + std::tie( consumed_source, tgt_leaf, tgt_leaf_next_pos ) = + merge + ( + *src_leaf, source_slot_offset, + *tgt_leaf, tgt_leaf_next_pos + ); + // TODO merge returns 'hints' for the next target position: use that + // for a faster version of find_nodes_for (that does not search from + // the beginning everytime). + + // merge might have caused a relocation (by calling split_to_insert) + // TODO use iter_pos directly + p_new_keys .update_pool_ptr( this->nodes_ ); + p_new_nodes_end.update_pool_ptr( this->nodes_ ); + p_new_keys += consumed_source; + } + + this->hdr().size_ += total_size; + } + + [[ using gnu: noinline, pure, sysv_abi ]] + const_iterator find( key_const_arg key ) const noexcept + { + if ( !empty() ) [[ likely ]] + { + auto const location{ const_cast( *this ).find_nodes_for( key ) }; + if ( location.leaf_offset.exact_find ) [[ likely ]] { + return iterator{ const_cast( *this ).nodes_, { slot_of( location.leaf ), location.leaf_offset.pos } }; + } } return this->cend(); @@ -1183,7 +1413,7 @@ class bp_tree [[ nodiscard ]] Comparator & mutable_comp() noexcept { return *this; } private: - struct find_pos + struct find_pos // msvc pass-in-reg facepalm { node_size_type pos : ( sizeof( node_size_type ) * CHAR_BIT - 1 ); node_size_type exact_find : 1; @@ -1230,12 +1460,12 @@ class bp_tree node_slot inner; }; - [[ gnu::hot, gnu::sysv_abi ]] + [[ using gnu: pure, hot, sysv_abi ]] key_locations find_nodes_for( key_const_arg key ) noexcept { node_slot separator_key_node; node_size_type separator_key_offset{}; - // if the root is a (lone) leaf_node is implicitly handled by the loop condition: + // a leaf (lone) root is implicitly handled by the loop condition: // depth_ == 1 so the loop is skipped entirely and the lone root is never examined // through the incorrectly typed reference auto p_node{ &bptree_base::as( root() ) }; @@ -1264,6 +1494,114 @@ class bp_tree }; } + // bulk insert helper: merge a new, presorted leaf into an existing leaf + auto merge + ( + leaf_node const & source, node_size_type const source_offset, + leaf_node & target, node_size_type const target_offset + ) noexcept + { + verify( source ); + verify( target ); + BOOST_ASSUME( source_offset < source.num_vals ); + node_size_type input_length ( source.num_vals - source_offset ); + node_size_type const available_space( target.max_values - target.num_vals ); + auto const src_keys{ &source.keys[ source_offset ] }; + auto & tgt_keys{ target.keys }; + if ( target_offset == 0 ) [[ unlikely ]] + { + auto const & new_separator{ src_keys[ 0 ] }; + BOOST_ASSUME( new_separator < tgt_keys[ 0 ] ); + base::update_separator( target, new_separator ); + // TODO rather simply insert the source leaf into the parent + } + if ( !available_space ) [[ unlikely ]] + { + auto const source_slot{ slot_of( source ) }; + BOOST_ASSERT( find( target, src_keys[ 0 ] ).pos == target_offset ); + auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, src_keys[ 0 ], {} ) }; + auto const & src{ base::leaf( source_slot ) }; + auto & tgt{ base::leaf( target_slot ) }; + BOOST_ASSUME( next_tgt_offset <= tgt.num_vals ); + auto const next_src_offset{ static_cast( source_offset + 1 ) }; + // next_tgt_offset returned by split_to_insert points to the + // position in the target node that immediately follows the + // position for the inserted src_keys[ 0 ] - IOW it need not be + // the position for src.keys[ next_src_offset ] + if ( tgt.num_vals != next_tgt_offset ) // necessary check because find assumes non-empty input + { + next_tgt_offset += find + ( + &tgt.keys[ next_tgt_offset ], + tgt.num_vals - next_tgt_offset, + src.keys[ next_src_offset ] + ).pos; + } + return std::make_tuple( 1, &tgt, next_tgt_offset ); + } + + auto copy_size{ std::min( input_length, available_space ) }; + // If there is an existing right sibling we must first check if the + // source contains values beyond its separator key (and adjust the copy + // size accordingly to maintain the sorted property). + if ( target.right ) + { + auto const & right_delimiter { base::leaf( target.right ).keys[ 0 ] }; + auto less_than_right_pos{ find( src_keys, copy_size, right_delimiter ).pos }; + if ( less_than_right_pos != copy_size ) + { + BOOST_ASSUME( less_than_right_pos < copy_size ); + auto const input_end_for_target{ less_than_right_pos + source_offset }; + BOOST_ASSUME( input_end_for_target > source_offset ); + BOOST_ASSUME( input_end_for_target <= source.num_vals ); + copy_size = input_end_for_target - source_offset; + } + } + + auto & tgt_size{ target.num_vals }; + if ( target_offset == tgt_size ) // a simple append + { + std::copy_n( src_keys, copy_size, &tgt_keys[ target_offset ] ); + tgt_size += copy_size; + } + else + { + BOOST_ASSUME( copy_size + tgt_size <= leaf_node::max_values ); + std::move_backward( &tgt_keys[ 0 ], &tgt_keys[ tgt_size ], &tgt_keys[ tgt_size + copy_size ] ); + tgt_size = merge_interleaved_values + ( + &src_keys[ 0 ], copy_size, + &tgt_keys[ copy_size ], tgt_size, + &tgt_keys[ 0 ] + ); + } + verify( target ); + return std::make_tuple( copy_size, &target, tgt_size ); + } + node_size_type merge_interleaved_values + ( + Key const source0[], node_size_type const source0_size, + Key const source1[], node_size_type const source1_size, + Key target [] + ) const noexcept + { + node_size_type const input_size( source0_size + source1_size ); + if constexpr ( unique ) + { + auto const out_pos{ std::set_union( source0, &source0[ source0_size ], source1, &source1[ source1_size ], target, comp() ) }; + auto const merged_size{ static_cast( out_pos - target ) }; + BOOST_ASSUME( merged_size <= input_size ); + return merged_size; + } + else + { + auto const out_pos{ std::merge( source0, &source0[ source0_size ], source1, &source1[ source1_size ], target, comp() ) }; + BOOST_ASSUME( out_pos = target + input_size ); + return input_size; + } + } + + [[ gnu::pure ]] bool le( key_const_arg left, key_const_arg right ) const noexcept { return comp()( left, right ); } [[ gnu::pure ]] bool eq( key_const_arg left, key_const_arg right ) const noexcept { diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index 374c29e..b62c2b3 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -15,10 +15,14 @@ namespace psi::vm // https://stackoverflow.com/questions/59362113/b-tree-minimum-internal-children-count-explanation // https://web.archive.org/web/20190126073810/http://supertech.csail.mit.edu/cacheObliviousBTree.html // https://www.researchgate.net/publication/220225482_Cache-Oblivious_Databases_Limitations_and_Opportunities +// https://www.postgresql.org/docs/current/btree.html +// https://abseil.io/about/design/btree // Data Structure Visualizations https://www.cs.usfca.edu/~galles/visualization/Algorithms.html // Griffin: Fast Transactional Database Index with Hash and B+-Tree https://ieeexplore.ieee.org/abstract/document/10678674 // Restructuring the concurrent B+-tree with non-blocked search operations https://www.sciencedirect.com/science/article/abs/pii/S002002550200261X +// https://en.wikipedia.org/wiki/Judy_array + bptree_base::bptree_base( header_info const hdr_info ) noexcept : nodes_{ hdr_info.add_header
() } @@ -44,10 +48,15 @@ bptree_base::init_header( storage_result success ) noexcept return success; } -void bptree_base::reserve( node_slot::value_type const additional_nodes ) +void bptree_base::reserve( node_slot::value_type additional_nodes ) { + auto const preallocated_count{ hdr().free_node_count_ }; + additional_nodes -= std::min( preallocated_count, additional_nodes ); auto const current_size{ nodes_.size() }; nodes_.grow_by( additional_nodes, value_init ); +#ifndef NDEBUG + hdr_ = &hdr(); +#endif for ( auto & n : std::views::reverse( std::span{ nodes_ }.subspan( current_size ) ) ) free( n ); } @@ -94,11 +103,11 @@ bptree_base::new_spillover_node_for( node_header & existing_node ) auto & left_node { node( existing_node_slot ) }; // handle relocation (by new_node()) auto const right_node_slot { slot_of( right_node ) }; - // insert into the level dlinked list + // insert into the level dlinked list (TODO extract into small specialized dlinked list functions) // node <- left/lesser | new_node -> right/greater right_node.left = existing_node_slot; right_node.right = left_node.right; - left_node.right = right_node_slot; + left_node.right = right_node_slot; update_right_sibling_link( right_node, right_node_slot ); right_node.parent = left_node.parent; right_node.parent_child_idx = left_node.parent_child_idx + 1; @@ -116,7 +125,7 @@ bptree_base::new_root( node_slot const left_child, node_slot const right_child ) auto & left { node( left_child ) }; BOOST_ASSUME( left.is_root() ); auto & right{ node( right_child ) }; new_root.left = new_root.right = {}; - new_root.num_vals = 1; + new_root.num_vals = 1; // the minimum of two children with one separator key hdr.root_ = slot_of( new_root ); left .parent = hdr.root_; right.parent = hdr.root_; @@ -126,29 +135,57 @@ bptree_base::new_root( node_slot const left_child, node_slot const right_child ) return new_root; } -bptree_base::base_iterator::base_iterator( node_pool & nodes, node_slot const node_offset, node_size_type const value_offset ) noexcept +bptree_base::base_iterator::base_iterator( node_pool & nodes, iter_pos const pos ) noexcept : + nodes_{}, + pos_{ pos } +{ + update_pool_ptr( nodes ); +} + +void bptree_base::base_iterator::update_pool_ptr( node_pool & nodes ) const noexcept +{ #ifndef NDEBUG // for bounds checking - nodes_{ nodes }, + nodes_ = nodes; #else - nodes_{ nodes.data() }, + nodes_ = nodes.data(); #endif - node_slot_{ node_offset }, value_offset_{ value_offset } -{} +} +[[ gnu::pure ]] bptree_base::node_header & -bptree_base::base_iterator::node() const noexcept { return nodes_[ *node_slot_ ]; } +bptree_base::base_iterator::node() const noexcept { return nodes_[ *pos_.node ]; } bptree_base::base_iterator & bptree_base::base_iterator::operator++() noexcept { - BOOST_ASSERT_MSG( node_slot_, "Iterator at end: not incrementable" ); auto & node{ this->node() }; + BOOST_ASSERT_MSG( pos_.value_offset < node.num_vals, "Iterator at end: not incrementable" ); BOOST_ASSUME( node.num_vals >= 1 ); - if ( ++value_offset_ == node.num_vals ) [[ unlikely ]] { - // implicitly becomes an end iterator when node.next is 'null' - node_slot_ = node.right; - value_offset_ = 0; + BOOST_ASSUME( pos_.value_offset < node.num_vals ); + if ( ++pos_.value_offset == node.num_vals ) [[ unlikely ]] + { + // have to perform this additional check to allow an iterator to arrive + // to the end position + if ( node.right ) [[ likely ]] + { + pos_.node = node.right; + pos_.value_offset = 0; + } + } + return *this; +} +bptree_base::base_iterator & +bptree_base::base_iterator::operator--() noexcept +{ + auto & node{ this->node() }; + BOOST_ASSERT_MSG( ( pos_.value_offset > 0 ) || node.left, "Iterator at end: not incrementable" ); + BOOST_ASSUME( pos_.value_offset <= node.num_vals ); + BOOST_ASSUME( node.num_vals >= 1 ); + if ( pos_.value_offset-- == 0 ) [[ unlikely ]] + { + pos_.node = node.left; + pos_.value_offset = this->node().num_vals - 1; } return *this; } @@ -156,30 +193,36 @@ bptree_base::base_iterator::operator++() noexcept bool bptree_base::base_iterator::operator==( base_iterator const & other ) const noexcept { BOOST_ASSERT_MSG( &this->nodes_[ 0 ] == &other.nodes_[ 0 ], "Comparing iterators from different containers" ); - return ( this->node_slot_ == other.node_slot_ ) && ( this->value_offset_ == other.value_offset_ ); + return ( this->pos_.node == other.pos_.node ) && ( this->pos_.value_offset == other.pos_.value_offset ); } bptree_base::base_random_access_iterator & bptree_base::base_random_access_iterator::operator+=( difference_type const n ) noexcept { - BOOST_ASSERT_MSG( node_slot_, "Iterator at end: not incrementable" ); - - if ( n >= 0 ) [[ likely ]] + BOOST_ASSERT_MSG( pos_.node, "Iterator at end: not incrementable" ); + + if ( n >= 0 ) { auto un{ static_cast( n ) }; for ( ;; ) { auto & node{ this->node() }; - auto const available_offset{ static_cast( node.num_vals - 1 - value_offset_ ) }; - if ( available_offset >= un ) [[ likely ]] { - value_offset_ += static_cast( un ); + auto const available_offset{ static_cast( node.num_vals - 1 - pos_.value_offset ) }; + if ( available_offset >= un ) { + pos_.value_offset += static_cast( un ); + BOOST_ASSUME( pos_.value_offset < node.num_vals ); break; } else { - un -= available_offset; - node_slot_ = node.right; - value_offset_ = 0; - BOOST_ASSERT_MSG( node_slot_ || un == 0, "Incrementing out of bounds" ); + un -= ( available_offset + 1 ); + // Here we don't have to perform the same check as in the + // fwd_iterator increment as (end) iterator comparison is done + // solely through the index_ member. + pos_.node = node.right; + pos_.value_offset = 0; + if ( un == 0 ) [[ unlikely ]] + break; + BOOST_ASSERT_MSG( pos_.node, "Incrementing out of bounds" ); } } index_ += static_cast( n ); @@ -190,21 +233,24 @@ bptree_base::base_random_access_iterator::operator+=( difference_type const n ) BOOST_ASSERT_MSG( index_ >= un, "Moving iterator out of bounds" ); for ( ;; ) { - auto & node{ this->node() }; - auto const available_offset{ value_offset_ }; - if ( available_offset >= un ) [[ likely ]] { - value_offset_ -= static_cast( un ); + auto const available_offset{ pos_.value_offset }; + if ( available_offset >= un ) { + pos_.value_offset -= static_cast( un ); + BOOST_ASSUME( pos_.value_offset < node().num_vals ); break; } else { - un -= available_offset; - node_slot_ = node.left; - value_offset_ = 0; - BOOST_ASSERT_MSG( node_slot_ || un == 0, "Incrementing out of bounds" ); + un -= ( available_offset + 1 ); + pos_.node = node().left; + pos_.value_offset = node().num_vals - 1; + BOOST_ASSERT_MSG( pos_.node, "Incrementing out of bounds" ); + BOOST_ASSUME( pos_.value_offset < node().num_vals ); } } - index_ += static_cast( -n ); + index_ -= static_cast( -n ); } + BOOST_ASSUME( !pos_.node || ( pos_.value_offset < node().num_vals ) ); + return *this; } @@ -213,8 +259,23 @@ void bptree_base::swap( bptree_base & other ) noexcept { using std::swap; swap( this->nodes_, other.nodes_ ); +#ifndef NDEBUG + swap( this->hdr_, other.hdr_ ); +#endif } +[[ gnu::pure ]] bptree_base::iter_pos bptree_base::begin_pos() const noexcept { return { this->first_leaf(), 0 }; } +[[ gnu::pure ]] bptree_base::iter_pos bptree_base:: end_pos() const noexcept { + auto const last_leaf{ hdr().last_leaf_ }; + return { last_leaf, node( last_leaf ).num_vals }; +} + +[[ gnu::pure ]] bptree_base::base_iterator bptree_base::begin() noexcept { return { this->nodes_, begin_pos() }; } +[[ gnu::pure ]] bptree_base::base_iterator bptree_base::end () noexcept { return { this->nodes_, end_pos() }; } + +[[ gnu::pure ]] bptree_base::base_random_access_iterator bptree_base::ra_begin() noexcept { return { *this, begin_pos(), 0 }; } +[[ gnu::pure ]] bptree_base::base_random_access_iterator bptree_base::ra_end () noexcept { return { *this, end_pos(), size() }; } + [[ gnu::cold ]] bptree_base::node_header & bptree_base::create_root() @@ -226,13 +287,14 @@ bptree_base::create_root() auto & root{ new_node() }; auto & hdr { this->hdr() }; BOOST_ASSUME( root.num_vals == 0 ); - root.num_vals = 1; - root.left = {}; - root.right = {}; - hdr.root_ = slot_of( root ); - hdr.leaves_ = hdr.root_; - hdr.depth_ = 1; - hdr.size_ = 1; + root.num_vals = 1; + root.left = {}; + root.right = {}; + hdr.root_ = slot_of( root ); + hdr.first_leaf_ = hdr.root_; + hdr.last_leaf_ = hdr.root_; + hdr.depth_ = 1; + hdr.size_ = 1; return root; } @@ -242,19 +304,21 @@ bptree_base::depth_t bptree_base:: leaf_level() const noexcept { BOOST_ASSUME( void bptree_base::free( node_header & node ) noexcept { BOOST_ASSUME( node.num_vals == 0 ); - auto & free_list{ hdr().free_list_ }; + auto & hdr{ this->hdr() }; + auto & free_list{ hdr.free_list_ }; auto & free_node{ static_cast( node ) }; -#ifndef NDEBUG auto const free_node_slot{ slot_of( free_node ) }; +#ifndef NDEBUG if ( free_node.left ) BOOST_ASSERT( this->node( free_node.left ).right != free_node_slot ); if ( free_node.right ) BOOST_ASSERT( this->node( free_node.right ).left != free_node_slot ); #endif free_node.left = {}; - if ( free_list ) free_node.right = free_list; - else free_node.right = {}; - free_list = slot_of( free_node ); + if ( free_list ) { BOOST_ASSUME( hdr.free_node_count_ ); free_node.right = free_list; this->node( free_list ).left = free_node_slot; } + else { BOOST_ASSUME( !hdr.free_node_count_ ); free_node.right = {} ; } + free_list = free_node_slot; + ++hdr.free_node_count_; } bptree_base::node_slot @@ -268,19 +332,27 @@ bptree_base::slot_of( node_header const & node ) const noexcept bptree_base::node_placeholder & bptree_base::new_node() { - auto & free_list{ hdr().free_list_ }; + auto & hdr { this->hdr() }; + auto & free_list{ hdr.free_list_ }; if ( free_list ) { + BOOST_ASSUME( hdr.free_node_count_ ); auto & cached_node{ node( free_list ) }; BOOST_ASSUME( !cached_node.num_vals ); BOOST_ASSUME( !cached_node.left ); free_list = cached_node.right; cached_node.right = {}; + if ( free_list ) + node( free_list ).left = {}; + --hdr.free_node_count_; return as( cached_node ); } auto & new_node{ nodes_.emplace_back() }; BOOST_ASSUME( new_node.num_vals == 0 ); new_node.left = new_node.right = {}; +#ifndef NDEBUG + hdr_ = &this->hdr(); +#endif return new_node; } diff --git a/test/b+tree.cpp b/test/b+tree.cpp index d31a4bc..0516287 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -20,36 +20,54 @@ static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) { #ifdef NDEBUG - auto const test_size{ 913735 }; + auto const test_size{ 9137365 }; #else - auto const test_size{ 93735 }; + auto const test_size{ 937358 }; #endif std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; std::mt19937 rng{ std::random_device{}() }; auto numbers{ std::ranges::to( sorted_numbers ) }; std::shuffle( numbers.begin(), numbers.end(), rng ); - { bp_tree bpt; bpt.map_memory(); bpt.reserve( numbers.size() ); - for ( auto const & n : numbers ) - bpt.insert( n ); - + { + auto const nums { std::span{ numbers } }; + auto const third{ nums.size() / 3 }; + auto const first_third{ nums.subspan( 0 * third, third ) }; + auto const second_third{ nums.subspan( 1 * third, third ) }; + auto const third_third{ nums.subspan( 2 * third ) }; + bpt.insert( first_third ); + for ( auto const & n : second_third ) { bpt.insert( n ); } + bpt.insert( third_third ); + } + + static_assert( std::forward_iterator::const_iterator> ); EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ) ) ); - EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); + EXPECT_TRUE( std::ranges::equal( sorted_numbers, bpt ) ); + EXPECT_TRUE( std::ranges::equal( sorted_numbers, bpt.random_access() ) ); EXPECT_NE( bpt.find( +42 ), bpt.end() ); EXPECT_EQ( bpt.find( -42 ), bpt.end() ); bpt.erase( 42 ); EXPECT_EQ( bpt.find( +42 ), bpt.end() ); bpt.insert( 42 ); + EXPECT_TRUE( std::ranges::equal( bpt.random_access() , sorted_numbers ) ); + EXPECT_TRUE( std::ranges::equal( std::views::reverse( bpt.random_access() ), std::views::reverse( sorted_numbers ) ) ); + { + auto const ra{ bpt.random_access() }; + for ( auto n : std::views::iota( 0, 45678 ) ) // slow operation: use a smaller subset of the input + EXPECT_EQ( ra[ n ], n ); + } std::shuffle( numbers.begin(), numbers.end(), rng ); for ( auto const & n : numbers ) bpt.erase( n ); + + EXPECT_TRUE( bpt.empty() ); } { From 7bc393cb8d224f7eb62c368328fa93ffb6cb00c5 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Fri, 11 Oct 2024 15:04:30 +0200 Subject: [PATCH 25/49] OSX: printf debugging. Decreased test data size (to avoid timeouts on GitHub). --- .github/workflows/gh-actions.yml | 2 +- test/b+tree.cpp | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index 728c820..e71cc6f 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -104,4 +104,4 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: > cd ${{ steps.strings.outputs.build-output-dir }}/test && - ctest --build-config ${{ matrix.build_type }} + ctest --progress --output-on-failure --build-config ${{ matrix.build_type }} diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 0516287..e09ba89 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -5,7 +5,7 @@ #include -#include +#include #include #include #include @@ -19,20 +19,22 @@ static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) { + std::println( "Generating test data." ); #ifdef NDEBUG - auto const test_size{ 9137365 }; + auto const test_size{ 1553735 }; #else - auto const test_size{ 937358 }; + auto const test_size{ 158735 }; #endif std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; std::mt19937 rng{ std::random_device{}() }; auto numbers{ std::ranges::to( sorted_numbers ) }; std::shuffle( numbers.begin(), numbers.end(), rng ); { + std::println( "B+tree in memory..." ); bp_tree bpt; bpt.map_memory(); bpt.reserve( numbers.size() ); - + std::println( " ...insertion" ); { auto const nums { std::span{ numbers } }; auto const third{ nums.size() / 3 }; @@ -44,7 +46,7 @@ TEST( bp_tree, playground ) bpt.insert( third_third ); } - + std::println( " ...checks" ); static_assert( std::forward_iterator::const_iterator> ); EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ) ) ); @@ -63,6 +65,7 @@ TEST( bp_tree, playground ) EXPECT_EQ( ra[ n ], n ); } + std::println( " ...erasure" ); std::shuffle( numbers.begin(), numbers.end(), rng ); for ( auto const & n : numbers ) bpt.erase( n ); @@ -71,6 +74,7 @@ TEST( bp_tree, playground ) } { + std::println( "B+tree on disk - creation..." ); bp_tree bpt; bpt.map_file( test_file, flags::named_object_construction_policy::create_new_or_truncate_existing ); @@ -87,6 +91,7 @@ TEST( bp_tree, playground ) EXPECT_EQ( bpt.find( +42 ), bpt.end() ); } { + std::println( "B+tree on disk - opening..." ); bp_tree bpt; bpt.map_file( test_file, flags::named_object_construction_policy::open_existing ); From c6b3134c2b0a03a6e54b069a2edef09865be01d8 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Fri, 11 Oct 2024 15:21:24 +0200 Subject: [PATCH 26/49] MSVC: fixed compiler warnings and a bogus assertion failure. --- include/psi/vm/containers/b+tree.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 8117bec..69632ea 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -456,7 +456,7 @@ class bptree_base_wkey : public bptree_base // TODO support for maps (i.e. keys+values) using value_type = Key; - static node_size_type constexpr storage_space{ node_size - align_up( sizeof( node_header ), alignof( Key ) ) }; + static node_size_type constexpr storage_space{ node_size - align_up( sizeof( node_header ), alignof( Key ) ) }; static node_size_type constexpr max_values { storage_space / sizeof( Key ) }; static node_size_type constexpr min_values { ihalf_ceil }; @@ -1014,7 +1014,7 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( left .right == slot_of( right ) ); BOOST_ASSUME( right.left == slot_of( left ) ); - std::ranges::move( keys( right ), keys( left ).end() ); + std::ranges::move( keys( right ), &keys( left ).back() + 1 ); left .num_vals += right.num_vals; right.num_vals = 0; BOOST_ASSUME( left.num_vals >= 2 * min - 2 ); @@ -1551,10 +1551,10 @@ class bp_tree if ( less_than_right_pos != copy_size ) { BOOST_ASSUME( less_than_right_pos < copy_size ); - auto const input_end_for_target{ less_than_right_pos + source_offset }; + node_size_type const input_end_for_target( less_than_right_pos + source_offset ); BOOST_ASSUME( input_end_for_target > source_offset ); BOOST_ASSUME( input_end_for_target <= source.num_vals ); - copy_size = input_end_for_target - source_offset; + copy_size = static_cast( input_end_for_target - source_offset ); } } From 5a77bc749194eacf13ed8757d0100d1e3ae7b7cc Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Fri, 11 Oct 2024 15:36:25 +0200 Subject: [PATCH 27/49] Fixed map_memory() to properly assign preallocated nodes to the free pool. --- include/psi/vm/containers/b+tree.hpp | 28 +++++++++++++++++++++++----- include/psi/vm/vector.hpp | 2 +- src/containers/b+tree.cpp | 15 ++++++--------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 69632ea..a6dc49f 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -85,8 +85,25 @@ class bptree_base bool has_attached_storage() const noexcept { return nodes_.has_attached_storage(); } - storage_result map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { return init_header( nodes_.map_file ( file, policy ) ); } - storage_result map_memory( std::uint32_t const initial_capacity_as_number_of_nodes = 0 ) noexcept { return init_header( nodes_.map_memory( initial_capacity_as_number_of_nodes ) ); } + storage_result map_file( auto const file, flags::named_object_construction_policy const policy ) noexcept + { + storage_result success{ nodes_.map_file( file, policy ) }; + if ( std::move( success ) && nodes_.empty() ) + hdr() = {}; + return success; + } + storage_result map_memory( std::uint32_t const initial_capacity_as_number_of_nodes = 0 ) noexcept + { + storage_result success{ nodes_.map_memory( initial_capacity_as_number_of_nodes ) }; + if ( std::move( success ) ) + { + hdr() = {}; + if ( initial_capacity_as_number_of_nodes ) { + assign_nodes_to_free_pool( 0 ); + } + } + return success; + } protected: static constexpr auto node_size{ page_size }; @@ -150,7 +167,7 @@ class bptree_base class base_iterator; class base_random_access_iterator; - struct header + struct header // or persisted data members { node_slot root_; node_slot first_leaf_; @@ -306,7 +323,7 @@ class bptree_base private: auto header_data() noexcept { return detail::header_data
( nodes_.user_header_data() ); } - storage_result init_header( storage_result ) noexcept; + void assign_nodes_to_free_pool( node_slot::value_type starting_node ) noexcept; protected: node_pool nodes_; @@ -739,8 +756,8 @@ class bptree_base_wkey : public bptree_base leaf_slot = leaf.right; else { - hdr.free_list_ = leaf.right; node( leaf.right ).left = {}; + hdr.free_list_ = leaf.right; leaf.right = {}; return std::make_pair( begin, iter_pos{ leaf_slot, size_to_copy } ); } @@ -1421,6 +1438,7 @@ class bp_tree [[ using gnu: pure, hot, noinline, sysv_abi ]] find_pos find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept { + // TODO branchless binary search, Alexandrescu's ideas, https://orlp.net/blog/bitwise-binary-search ... BOOST_ASSUME( num_vals > 0 ); auto const & __restrict comp{ this->comp() }; node_size_type pos_idx; diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index 8efaa5c..2c7953f 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -1084,7 +1084,7 @@ class vector /////////////////////////////////////////////////////////////////////////// auto map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_file ( file, policy ); } - auto map_memory( size_type const initial_capacity = 0 ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_memory( to_byte_sz( initial_capacity ) ); } + auto map_memory( size_type const initial_size = 0 ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_memory( to_byte_sz( initial_size ) ); } bool has_attached_storage() const noexcept { return static_cast( storage_ ); } diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index b62c2b3..0db01fe 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -40,14 +40,6 @@ bptree_base::user_header_data() noexcept { return header_data().second; } bptree_base::header & bptree_base::hdr() noexcept { return *header_data().first; } -bptree_base::storage_result -bptree_base::init_header( storage_result success ) noexcept -{ - if ( std::move( success ) && nodes_.empty() ) - hdr() = {}; - return success; -} - void bptree_base::reserve( node_slot::value_type additional_nodes ) { auto const preallocated_count{ hdr().free_node_count_ }; @@ -57,7 +49,12 @@ void bptree_base::reserve( node_slot::value_type additional_nodes ) #ifndef NDEBUG hdr_ = &hdr(); #endif - for ( auto & n : std::views::reverse( std::span{ nodes_ }.subspan( current_size ) ) ) + assign_nodes_to_free_pool( static_cast( current_size ) ); +} + +void bptree_base::assign_nodes_to_free_pool( node_slot::value_type const starting_node ) noexcept +{ + for ( auto & n : std::views::reverse( std::span{ nodes_ }.subspan( starting_node ) ) ) free( n ); } From b399d7c37d6b9f19f1d6e10bf9ed39f50a63dae7 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sun, 13 Oct 2024 22:39:26 +0200 Subject: [PATCH 28/49] Increased release test iterations again. Removed 'printf debugging' statements. --- test/b+tree.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/b+tree.cpp b/test/b+tree.cpp index e09ba89..f12dd65 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -5,7 +5,6 @@ #include -#include #include #include #include @@ -19,9 +18,8 @@ static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) { - std::println( "Generating test data." ); #ifdef NDEBUG - auto const test_size{ 1553735 }; + auto const test_size{ 3553735 }; #else auto const test_size{ 158735 }; #endif @@ -30,11 +28,9 @@ TEST( bp_tree, playground ) auto numbers{ std::ranges::to( sorted_numbers ) }; std::shuffle( numbers.begin(), numbers.end(), rng ); { - std::println( "B+tree in memory..." ); bp_tree bpt; bpt.map_memory(); bpt.reserve( numbers.size() ); - std::println( " ...insertion" ); { auto const nums { std::span{ numbers } }; auto const third{ nums.size() / 3 }; @@ -46,7 +42,6 @@ TEST( bp_tree, playground ) bpt.insert( third_third ); } - std::println( " ...checks" ); static_assert( std::forward_iterator::const_iterator> ); EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ) ) ); @@ -65,7 +60,6 @@ TEST( bp_tree, playground ) EXPECT_EQ( ra[ n ], n ); } - std::println( " ...erasure" ); std::shuffle( numbers.begin(), numbers.end(), rng ); for ( auto const & n : numbers ) bpt.erase( n ); @@ -74,7 +68,6 @@ TEST( bp_tree, playground ) } { - std::println( "B+tree on disk - creation..." ); bp_tree bpt; bpt.map_file( test_file, flags::named_object_construction_policy::create_new_or_truncate_existing ); @@ -91,7 +84,6 @@ TEST( bp_tree, playground ) EXPECT_EQ( bpt.find( +42 ), bpt.end() ); } { - std::println( "B+tree on disk - opening..." ); bp_tree bpt; bpt.map_file( test_file, flags::named_object_construction_policy::open_existing ); From c0239f3e164e476b77034543a893bb31560d3b05 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sun, 13 Oct 2024 22:40:10 +0200 Subject: [PATCH 29/49] Minor stylistic fix. --- src/allocation/remap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allocation/remap.cpp b/src/allocation/remap.cpp index bdac147..2fdcfab 100644 --- a/src/allocation/remap.cpp +++ b/src/allocation/remap.cpp @@ -148,7 +148,7 @@ expand_result expand_back expand_result expand_front ( - std::byte * const address, + std::byte * const address, std::size_t const current_size, std::size_t const required_size, std::size_t const used_capacity, From 214f6946cfb6887085e0cea8d35bb910db1de6ac Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sun, 13 Oct 2024 22:45:29 +0200 Subject: [PATCH 30/49] POSIX: added a quick-fix for non-linux systems for expanding an anonymous mappings that cannot be done in-place. Linux: disabled/excluded the POSIX fallback in-case-mremap-fails (relying solely on mremap now). --- src/mapped_view/mapped_view.cpp | 66 ++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/mapped_view/mapped_view.cpp b/src/mapped_view/mapped_view.cpp index 0341821..0c28283 100644 --- a/src/mapped_view/mapped_view.cpp +++ b/src/mapped_view/mapped_view.cpp @@ -13,6 +13,8 @@ //------------------------------------------------------------------------------ #include #include + +#include // memcpy //------------------------------------------------------------------------------ namespace psi::vm { @@ -89,9 +91,11 @@ template fallible_result basic_mapped_view::expand( std::size_t const target_size, mapping & original_mapping ) noexcept { - auto const current_address { const_cast( this->data() ) }; - auto const current_size { this->size() }; - auto const kernel_current_size{ align_up( current_size, commit_granularity ) }; + // TODO kill duplication with remap.cpp::expand() + + auto const current_address { const_cast( this->data() ) }; + auto const current_size { this->size() }; + auto const kernel_current_size{ align_up( current_size, commit_granularity ) }; if ( kernel_current_size >= target_size ) { BOOST_ASSUME( current_address ); @@ -110,12 +114,27 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o static_cast( *this ) = { static_cast( new_address ), target_size }; return err::success; } -#endif // linux mremap + return error_t{}; +#else auto const current_offset{ 0U }; // TODO: what if this is an offset view? auto const target_offset { current_offset + kernel_current_size }; auto const additional_tail_size{ target_size - kernel_current_size }; auto const tail_target_address { current_address + kernel_current_size }; -#ifndef _WIN32 +#ifdef _WIN32 + ULARGE_INTEGER const win32_offset{ .QuadPart = target_offset }; + auto const new_address + { + ::MapViewOfFileEx + ( + original_mapping.get(), + original_mapping.view_mapping_flags.map_view_flags, + win32_offset.HighPart, + win32_offset.LowPart, + additional_tail_size, + tail_target_address + ) + }; +#else auto new_address { posix::mmap @@ -131,34 +150,20 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o if ( ( new_address != tail_target_address ) && // On POSIX the target address is only a hint (while MAP_FIXED may overwrite existing mappings) - ( current_address != nullptr ) // in case of starting with an empty/null/zero-sized views (is it worth the special handling?) + ( current_address != nullptr ) // in case of starting with an empty/null/zero-sized view (is it worth the special handling?) ) { BOOST_VERIFY( ::munmap( new_address, additional_tail_size ) == 0 ); new_address = nullptr; } -#else - ULARGE_INTEGER const win32_offset{ .QuadPart = target_offset }; - auto const new_address - { - ::MapViewOfFileEx - ( - original_mapping.get(), - original_mapping.view_mapping_flags.map_view_flags, - win32_offset.HighPart, - win32_offset.LowPart, - additional_tail_size, - tail_target_address - ) - }; #endif if ( new_address ) [[ likely ]] { if ( current_address != nullptr ) [[ likely ]] { BOOST_ASSUME( new_address == current_address + kernel_current_size ); -# ifdef __linux__ - BOOST_ASSERT_MSG( false, "mremap failed but an overlapping mmap succeeded!?" ); // behaviour investigation +# ifdef __linux__ // no longer exercised path (i.e. linux only relies on mremap) + BOOST_ASSERT_MSG( false, "mremap failed but an adjacent mmap succeeded!?" ); // behaviour investigation # endif static_cast( *this ) = { current_address, target_size }; } @@ -172,8 +177,25 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o auto remapped_span{ this->map( original_mapping, 0, target_size )() }; if ( !remapped_span ) return remapped_span.error(); +#ifndef _WIN32 + // Linux handles relocation-on-expansion implicitly with mremap (it even + // supports creating new mappings of the same pages/physical memory by + // specifying zero for old_size -> TODO use this to implement 'multimappable' + // mappings, 'equivalents' of NtCreateSection instead of going through shm), + // Windows tracks the source data/backing storage through the mapping + // handle. For others we have to the the new-copy-free old dance - but this + // is a quick-fix as this still does not support 'multiply remappable + // mapping' semantics a la NtCreateSection - TODO: shm_mkstemp, SHM_ANON, + // memfd_create... + // https://github.com/lassik/shm_open_anon + if ( !original_mapping.is_file_based() ) + std::memcpy( const_cast( remapped_span->data() ), this->data(), this->size() ); + else +#endif + BOOST_ASSERT_MSG( std::memcmp( remapped_span->data(), this->data(), this->size() ) == 0, "View expansion garbled data." ); *this = std::move( *remapped_span ); return err::success; +#endif // end of 'no linux mremap' implementation } // view::expand() template class basic_mapped_view; From ab96b1fa930c201667b54505fbfefce45853db16 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sun, 13 Oct 2024 22:47:22 +0200 Subject: [PATCH 31/49] Expanded the vector.anon_memory_backed test to also verify that expansion (with relocation) does not destroy the preexisting contents. --- test/vector.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/vector.cpp b/test/vector.cpp index f3bacde..3be0649 100644 --- a/test/vector.cpp +++ b/test/vector.cpp @@ -10,7 +10,7 @@ namespace psi::vm TEST( vector, anon_memory_backed ) { - psi::vm::vector vec; + psi::vm::vector vec; vec.map_memory(); EXPECT_EQ( vec.size(), 0 ); vec.append_range({ 3.14, 0.14, 0.04 }); @@ -18,6 +18,11 @@ TEST( vector, anon_memory_backed ) EXPECT_EQ( vec[ 0 ], 3.14 ); EXPECT_EQ( vec[ 1 ], 0.14 ); EXPECT_EQ( vec[ 2 ], 0.04 ); + vec.grow_by( 12345678, default_init ); + // test growth (with 'probable' relocation) does not destroy contents + EXPECT_EQ( vec[ 0 ], 3.14 ); + EXPECT_EQ( vec[ 1 ], 0.14 ); + EXPECT_EQ( vec[ 2 ], 0.04 ); } TEST( vector, file_backed ) From 07d8ba712b31b9f57d7330e3cfaed149ee2bd4ea Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Mon, 14 Oct 2024 00:17:24 +0200 Subject: [PATCH 32/49] Linux: fixed a recently introduced bug in basic_mapped_view<>::expand() (now that there is no fallback for mremap we have to add explicit handling of initial mapping with an explicit/special-cased mmap call). --- src/mapped_view/mapped_view.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/mapped_view/mapped_view.cpp b/src/mapped_view/mapped_view.cpp index 0c28283..a12b6ff 100644 --- a/src/mapped_view/mapped_view.cpp +++ b/src/mapped_view/mapped_view.cpp @@ -105,6 +105,31 @@ basic_mapped_view::expand( std::size_t const target_size, mapping & o } #if defined( __linux__ ) + // unlike realloc mremap does not support functioning (also) as 'malloc' + // i.e. it does not support nullptr as the 'old address' + if ( !current_address ) [[ unlikely ]] // initial mapping + { + BOOST_ASSUME( !current_size ); + auto initial_address + { + posix::mmap + ( + nullptr, + target_size, + original_mapping.view_mapping_flags.protection, + original_mapping.view_mapping_flags.flags, + original_mapping.get(), + 0 + ) + }; + if ( initial_address ) [[ likely ]] + { + static_cast( *this ) = { static_cast( initial_address ), target_size }; + return err::success; + } + return error_t{}; + } + if ( auto const new_address{ ::mremap( current_address, current_size, target_size, std::to_underlying( reallocation_type::moveable ) ) }; From cca72fc98657c734c6da609157077acf99161385 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Mon, 14 Oct 2024 00:18:26 +0200 Subject: [PATCH 33/49] Clang: silenced 'expected' ubsan warnings. --- include/psi/vm/containers/b+tree.hpp | 1 + src/containers/b+tree.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index a6dc49f..b5d32c3 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -390,6 +390,7 @@ class bptree_base::base_random_access_iterator : public base_iterator public: constexpr base_random_access_iterator() noexcept = default; + [[ clang::no_sanitize( "unsigned-integer-overflow" ) ]] difference_type operator-( base_random_access_iterator const & other ) const noexcept { return static_cast( this->index_ - other.index_ ); } base_random_access_iterator & operator+=( difference_type n ) noexcept; diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index 0db01fe..7d766a9 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -172,6 +172,7 @@ bptree_base::base_iterator::operator++() noexcept } return *this; } +[[ clang::no_sanitize( "implicit-conversion" ) ]] bptree_base::base_iterator & bptree_base::base_iterator::operator--() noexcept { From cb21b2be24f559035fa6a66c7acc6edcb6392048 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Thu, 17 Oct 2024 12:25:00 +0200 Subject: [PATCH 34/49] Added the public bp_tree::merge() for efficient merging of two trees. Fixed a (node) leak in the bulk version of insert(). Fixed logic in insert (and the new merge) for maintaining the minimum-number-of-children-per-node rule (at the beginning and end of the bulk_append phase). Changed unlink_node() to also free the node. --- include/psi/vm/containers/b+tree.hpp | 379 ++++++++++++++++++++------- src/containers/b+tree.cpp | 86 ++++-- test/b+tree.cpp | 57 +++- 3 files changed, 385 insertions(+), 137 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index b5d32c3..7fdc295 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -124,34 +124,39 @@ class bptree_base [[ gnu::pure ]] explicit operator bool() const noexcept { return index != null.index; } }; // struct node_slot - struct [[ nodiscard ]] node_header + struct [[ nodiscard, clang::trivial_abi ]] node_header { using size_type = std::uint16_t; - // At minimum we need single-linked/directed list in the vertical/depth and horizontal/breadth directions - // (and the latter only for the leaf level - to have a connected sorted 'list' of all the values). + // At minimum we need a single-linked/directed list in the vertical/depth + // and horizontal/breadth directions (and the latter only for the leaf + // level - to have a connected sorted 'list' of all the values). // However having a precise vertical back/up-link (parent_child_idx): - // * speeds up walks up the tree (as the parent (separator) key slots do not have to be searched for) - // * simplifies code (enabling several functions to become independent of the comparator - no longer - // need searching - and moved up into the base bptree classes) - // * while at the same time being a negligible overhead considering we are targeting much larger (page - // size sized) nodes. + // * speeds up walks up the tree (as the parent (separator) key slots do + // not have to be searched for) + // * simplifies code (enabling several functions to become independent + // of the comparator - no longer need searching - and moved up into + // the base bptree classes) + // * while at the same time being a negligible overhead considering we + // are targeting much larger (page size sized) nodes. node_slot parent {}; node_slot left {}; node_slot right {}; size_type num_vals {}; size_type parent_child_idx{}; /* TODO - size_type start; // make keys and children arrays function as devectors: allow empty space at the beginning to avoid moves for small borrowings + size_type start; // make keys and children arrays function as devectors: allow empty space at the beginning to avoid moves for smaller borrowings */ [[ gnu::pure ]] bool is_root() const noexcept { return !parent; } - // merely to prevent slicing (in return-node-by-ref cases) - node_header( node_header const & ) = delete; - constexpr node_header( node_header && ) noexcept = default; - constexpr node_header( ) noexcept = default; - constexpr node_header & operator=( node_header && ) noexcept = default; + protected: // merely to prevent slicing (in return-node-by-ref cases) + constexpr node_header( node_header const & ) noexcept = default; + constexpr node_header( node_header && ) noexcept = default; + public: + constexpr node_header( ) noexcept = default; + constexpr node_header & operator=( node_header && ) noexcept = default; + constexpr node_header & operator=( node_header const & ) noexcept = default; }; // struct node_header using node_size_type = node_header::size_type; @@ -167,6 +172,8 @@ class bptree_base class base_iterator; class base_random_access_iterator; + struct insert_pos_t { node_slot node; node_size_type next_insert_offset; }; // TODO 'merge' with iter_pos + struct header // or persisted data members { node_slot root_; @@ -213,7 +220,6 @@ class bptree_base static void verify( auto const & node ) noexcept { - BOOST_ASSERT( std::ranges::is_sorted( keys( node ) ) ); BOOST_ASSUME( node.num_vals <= node.max_values ); // also used for underflowing nodes and (most problematically) for root nodes 'interpreted' as inner nodes...TODO... //BOOST_ASSUME( node.num_vals >= node.min_values ); @@ -285,6 +291,10 @@ class bptree_base void update_right_sibling_link( node_header const & left_node, node_slot left_node_slot ) noexcept; void unlink_node( node_header & node, node_header & cached_left_sibling ) noexcept; + void unlink_left ( node_header & nd ) noexcept; + void unlink_right( node_header & nd ) noexcept; + void link( node_header & left, node_header & right ) const noexcept; + [[ gnu::sysv_abi ]] std::pair new_spillover_node_for( node_header & existing_node ); @@ -293,7 +303,7 @@ class bptree_base template static NodeType & as( SourceNode & slot ) noexcept { - static_assert( sizeof( NodeType ) == sizeof( slot ) ); + static_assert( sizeof( NodeType ) == sizeof( slot ) || std::is_same_v ); return static_cast( static_cast( slot ) ); } template @@ -308,7 +318,13 @@ class bptree_base template N & node( node_slot const offset ) noexcept { return as( node( offset ) ); } template N const & node( node_slot const offset ) const noexcept { return as( node( offset ) ); } - [[ gnu::pure ]] node_slot slot_of( node_header const & ) const noexcept; + template N & right( N const & nd ) noexcept { return node( nd.right ); } + template N const & right( N const & nd ) const noexcept { return node( nd.right ); } + template N & left ( N const & nd ) noexcept { return node( nd.left ); } + template N const & left ( N const & nd ) const noexcept { return node( nd.left ); } + + [[ gnu::pure ]] bool is_my_node( node_header const & ) const noexcept; + [[ gnu::pure ]] node_slot slot_of ( node_header const & ) const noexcept; static bool full( auto const & node ) noexcept { BOOST_ASSUME( node.num_vals <= node.max_values ); @@ -369,7 +385,7 @@ class bptree_base::base_iterator bool operator==( base_iterator const & ) const noexcept; public: - auto pos() const noexcept { return pos_; } + iter_pos const & pos() const noexcept { return pos_; } }; // class base_iterator //////////////////////////////////////////////////////////////////////////////// @@ -499,8 +515,6 @@ class bptree_base_wkey : public bptree_base class ra_iterator; protected: // split_to_insert and its helpers - struct insert_pos_t { node_slot node; node_size_type next_insert_offset; }; - void new_root( node_slot const left_child, node_slot const right_child, key_const_arg separator_key ) { auto & new_root_node{ as( bptree_base::new_root( left_child, right_child ) ) }; @@ -716,12 +730,35 @@ class bptree_base_wkey : public bptree_base } } - void bulk_append( leaf_node const * src_leaf, insert_pos_t rightmost_parent_pos ) noexcept + // This function only serves the purpose of maintaining the rule about the + // minimum number of children per node - that rule is actually only + // 'academic' (for making sure that the performance/complexity guarantees + // will be maintained) - the tree would operate correctly even without + // maintaining that invariant (TODO make this an option). + bool bulk_append_fill_incomplete_leaf( leaf_node & leaf ) noexcept + { + auto const missing_keys{ static_cast( std::max( 0, signed( leaf.min_values ) - leaf.num_vals ) ) }; + if ( missing_keys ) + { + auto & preceding{ left( leaf ) }; + BOOST_ASSUME( preceding.num_vals + leaf.num_vals >= leaf_node::min_values * 2 ); + std::shift_right( &leaf.keys[ 0 ], &leaf.keys[ leaf.num_vals + missing_keys ], missing_keys ); + this->move_keys( preceding, preceding.num_vals - missing_keys, preceding.num_vals, leaf, 0 ); + leaf .num_vals += missing_keys; + preceding.num_vals -= missing_keys; + return true; + } + return false; + } + + void bulk_append( leaf_node * src_leaf, insert_pos_t rightmost_parent_pos ) { for ( ;; ) { + BOOST_ASSUME( !src_leaf->parent ); auto & rightmost_parent{ inner( rightmost_parent_pos.node ) }; BOOST_ASSUME( rightmost_parent_pos.next_insert_offset == rightmost_parent.num_vals ); + auto const next_src_slot{ src_leaf->right }; rightmost_parent_pos = insert ( rightmost_parent, @@ -729,11 +766,22 @@ class bptree_base_wkey : public bptree_base src_leaf->keys[ 0 ], slot_of( *src_leaf ) ); - if ( !src_leaf->right ) + if ( !next_src_slot ) break; - src_leaf = &leaf( src_leaf->right ); + src_leaf = &leaf( next_src_slot ); } hdr().last_leaf_ = slot_of( *src_leaf ); + if ( bulk_append_fill_incomplete_leaf( *src_leaf ) ) + { + // Borrowing from the left sibling happened _after_ src_leaf was + // already inserted into the parent so we have to update the + // separator key in the parent (since this is the rightmost leaf we + // know that the separator key has to be in the immediate parent - + // no need to call the generic update_separator()). + auto & prnt{ parent( *src_leaf ) }; + BOOST_ASSUME( src_leaf->parent_child_idx == num_chldrn( prnt ) - 1 ); + keys( prnt ).back() = src_leaf->keys[ 0 ]; + } } auto bulk_insert_prepare( std::span keys ) @@ -757,9 +805,8 @@ class bptree_base_wkey : public bptree_base leaf_slot = leaf.right; else { - node( leaf.right ).left = {}; hdr.free_list_ = leaf.right; - leaf.right = {}; + unlink_right( leaf ); return std::make_pair( begin, iter_pos{ leaf_slot, size_to_copy } ); } } @@ -822,9 +869,33 @@ class bptree_base_wkey : public bptree_base parent_key = new_separator; } + void move( leaf_node & source, leaf_node & target ) noexcept + { + BOOST_ASSUME( &source != &target ); + BOOST_ASSUME( source.num_values >= leaf_node::min_vals ); + BOOST_ASSUME( source.num_values <= leaf_node::max_vals ); + + auto constexpr copy_whole{ std::is_trivial_v && false }; + if constexpr ( copy_whole ) + { + std::memcpy( &target, &source, sizeof( target ) ); + } + else + if constexpr ( std::is_trivial_v ) + { + std::memcpy( &target, &source, reinterpret_cast( &source.keys[ source.num_vals ] ) - reinterpret_cast( &source ) ); + } + else + { + std::ranges::uninitialized_move( keys( source ), target.keys ); + static_cast( target ) = std::move( source ); + source.num_vals = 0; + } + } + template BOOST_NOINLINE - void handle_underflow( N & node, depth_t const level ) + void handle_underflow( N & node, depth_t const level ) noexcept { BOOST_ASSUME( underflowed( node ) ); @@ -848,8 +919,8 @@ class bptree_base_wkey : public bptree_base // (and so cannot be used to resolve existence of siblings) auto const has_right_sibling{ parent_child_idx < ( num_chldrn( parent ) - 1 ) }; auto const has_left_sibling { parent_child_idx > 0 }; - auto const p_right_sibling { has_right_sibling ? &this->node( node.right ) : nullptr }; - auto const p_left_sibling { has_left_sibling ? &this->node( node.left ) : nullptr }; + auto const p_right_sibling { has_right_sibling ? &right( node ) : nullptr }; + auto const p_left_sibling { has_left_sibling ? &left ( node ) : nullptr }; auto const right_separator_key_idx{ static_cast( parent_key_idx + parent_has_key_copy ) }; auto const left_separator_key_idx{ std::min( static_cast( right_separator_key_idx - 1 ), parent.num_vals ) }; // (ab)use unsigned wraparound @@ -980,14 +1051,11 @@ class bptree_base_wkey : public bptree_base using bptree_base::free; void free( leaf_node & leaf ) noexcept { auto & first_leaf{ hdr().first_leaf_ }; - if ( first_leaf == slot_of( leaf ) ) { + if ( first_leaf == slot_of( leaf ) ) + { BOOST_ASSUME( !leaf.left ); first_leaf = leaf.right; - if ( first_leaf ) - { - BOOST_ASSERT( node( first_leaf ).left ); - node( first_leaf ).left = {}; - } + unlink_right( leaf ); } bptree_base::free( static_cast( leaf ) ); } @@ -1043,10 +1111,7 @@ class bptree_base_wkey : public bptree_base parent.num_vals--; unlink_node( right, left ); - - right.right = {}; verify( left ); - free ( right ); verify( parent ); } [[ gnu::sysv_abi ]] @@ -1059,7 +1124,6 @@ class bptree_base_wkey : public bptree_base auto constexpr min{ inner_node::min_values }; BOOST_ASSUME( right.num_vals >= min - 1 ); BOOST_ASSUME( right.num_vals <= min ); BOOST_ASSUME( left .num_vals >= min - 1 ); BOOST_ASSUME( left .num_vals <= min ); - unlink_node( right, left ); move_chldrn( right, 0, num_chldrn( right ), left, num_chldrn( left ) ); auto & separator_key{ parent.keys[ parent_key_idx ] }; @@ -1067,11 +1131,10 @@ class bptree_base_wkey : public bptree_base keys( left ).back() = std::move( separator_key ); std::ranges::move( keys( right ), keys( left ).end() ); - left .num_vals += right.num_vals; - right.num_vals = 0; + left.num_vals += right.num_vals; BOOST_ASSUME( left.num_vals >= left.max_values - 1 ); BOOST_ASSUME( left.num_vals <= left.max_values ); - verify( left ); - free ( right ); + verify( left ); + unlink_node( right, left ); lshift_keys ( parent, parent_key_idx ); lshift_chldrn( parent, parent_child_idx ); @@ -1102,7 +1165,8 @@ class bptree_base_wkey::fwd_iterator { auto & leaf{ static_cast( node() ) }; BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); - return leaf.keys[ pos_.value_offset ]; } + return leaf.keys[ pos_.value_offset ]; + } constexpr fwd_iterator & operator++() noexcept { return static_cast( base_iterator::operator++() ); } constexpr fwd_iterator & operator--() noexcept { return static_cast( base_iterator::operator--() ); } @@ -1157,7 +1221,7 @@ void bptree_base_wkey::move_keys { BOOST_ASSUME( &source != &target ); // otherwise could require move_backwards or shift_* BOOST_ASSUME( src_begin <= src_end ); - BOOST_ASSUME( ( src_end - src_begin ) <= N::min_values + 1 ); + BOOST_ASSUME( ( src_end - src_begin ) <= N::max_values ); BOOST_ASSUME( tgt_begin < N::max_values ); std::uninitialized_move( &source.keys[ src_begin ], &source.keys[ src_end ], &target.keys[ tgt_begin ] ); } @@ -1293,62 +1357,9 @@ class bp_tree // bulk insert // TODO proper std insert interface (w/ ranges, iterators, hints...) - // https://www.sciencedirect.com/science/article/abs/pii/S0020025502002025 On batch-constructing B+-trees: algorithm and its performance - // evaluation https://www.vldb.org/conf/2001/P461.pdf An Evaluation of Generic Bulk Loading Techniques - // https://stackoverflow.com/questions/15996319/is-there-any-algorithm-for-bulk-loading-in-b-tree - void insert( std::span keys ) - { - auto const total_size{ static_cast( keys.size() ) }; - auto const [begin_leaf, end_pos]{ base::bulk_insert_prepare( keys ) }; - ra_iterator const p_new_nodes_begin{ *this, { begin_leaf, 0 }, 0 }; - ra_iterator const p_new_nodes_end { *this, end_pos , total_size }; - std::sort( p_new_nodes_begin, p_new_nodes_end, comp() ); + void insert( std::span keys ); - if ( empty() ) - return base::bulk_insert_into_empty( begin_leaf, end_pos, total_size ); - - for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) - { - auto [ source_slot, source_slot_offset ]{ p_new_keys.pos() }; - auto const tgt_location{ find_nodes_for( *p_new_keys ) }; - auto tgt_leaf { &tgt_location.leaf }; - auto tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; - - auto * src_leaf{ &base::leaf( source_slot ) }; - // if we have reached the end of the rightmost leaf simply perform - // a bulk_append - if ( ( tgt_leaf_next_pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) - { - lshift_keys( *src_leaf, source_slot_offset ); - src_leaf->num_vals -= source_slot_offset; - src_leaf->left = slot_of( *tgt_leaf ); - tgt_leaf->right = source_slot; - auto const rightmost_parent_slot{ tgt_leaf->parent }; - auto const parent_pos{ tgt_leaf->parent_child_idx }; // key idx = child idx - 1 & this is the 'next' key - base::bulk_append( src_leaf, { rightmost_parent_slot, parent_pos } ); - break; - } - - node_size_type consumed_source; - std::tie( consumed_source, tgt_leaf, tgt_leaf_next_pos ) = - merge - ( - *src_leaf, source_slot_offset, - *tgt_leaf, tgt_leaf_next_pos - ); - // TODO merge returns 'hints' for the next target position: use that - // for a faster version of find_nodes_for (that does not search from - // the beginning everytime). - - // merge might have caused a relocation (by calling split_to_insert) - // TODO use iter_pos directly - p_new_keys .update_pool_ptr( this->nodes_ ); - p_new_nodes_end.update_pool_ptr( this->nodes_ ); - p_new_keys += consumed_source; - } - - this->hdr().size_ += total_size; - } + void merge( bp_tree & other ); [[ using gnu: noinline, pure, sysv_abi ]] const_iterator find( key_const_arg key ) const noexcept @@ -1536,11 +1547,12 @@ class bp_tree } if ( !available_space ) [[ unlikely ]] { - auto const source_slot{ slot_of( source ) }; + // support merging nodes from another tree instance + auto const source_slot{ base::is_my_node( source ) ? slot_of( source ) : node_slot{} }; BOOST_ASSERT( find( target, src_keys[ 0 ] ).pos == target_offset ); auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, src_keys[ 0 ], {} ) }; - auto const & src{ base::leaf( source_slot ) }; - auto & tgt{ base::leaf( target_slot ) }; + auto const & src{ source_slot ? base::leaf( source_slot ) : source }; + auto & tgt{ base::leaf( target_slot ) }; BOOST_ASSUME( next_tgt_offset <= tgt.num_vals ); auto const next_src_offset{ static_cast( source_offset + 1 ) }; // next_tgt_offset returned by split_to_insert points to the @@ -1620,6 +1632,14 @@ class bp_tree } } +#if !( defined( _MSC_VER ) && !defined( __clang__ ) ) + // ambiguous call w/ VS 17.11.5 + void verify( auto const & node ) const noexcept + { + BOOST_ASSERT( std::ranges::is_sorted( keys( node ), comp() ) ); + base::verify( node ); + } +#endif [[ gnu::pure ]] bool le( key_const_arg left, key_const_arg right ) const noexcept { return comp()( left, right ); } [[ gnu::pure ]] bool eq( key_const_arg left, key_const_arg right ) const noexcept @@ -1642,6 +1662,161 @@ class bp_tree PSI_WARNING_DISABLE_POP() +template +void bp_tree::insert( std::span keys ) +{ + // https://www.sciencedirect.com/science/article/abs/pii/S0020025502002025 On batch-constructing B+-trees: algorithm and its performance + // https://www.vldb.org/conf/2001/P461.pdf An Evaluation of Generic Bulk Loading Techniques + // https://stackoverflow.com/questions/15996319/is-there-any-algorithm-for-bulk-loading-in-b-tree + + auto const total_size{ static_cast( keys.size() ) }; + auto const [begin_leaf, end_pos]{ base::bulk_insert_prepare( keys ) }; + ra_iterator const p_new_nodes_begin{ *this, { begin_leaf, 0 }, 0 }; + ra_iterator const p_new_nodes_end { *this, end_pos , total_size }; + std::sort( p_new_nodes_begin, p_new_nodes_end, comp() ); + + if ( empty() ) + return base::bulk_insert_into_empty( begin_leaf, end_pos, total_size ); + + for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) + { + auto const tgt_location{ find_nodes_for( *p_new_keys ) }; + BOOST_ASSUME( !tgt_location.leaf_offset.exact_find ); + auto tgt_leaf { &tgt_location.leaf }; + auto tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; + + auto const [source_slot, source_slot_offset]{ p_new_keys.pos() }; + auto * src_leaf{ &this->leaf( source_slot ) }; + BOOST_ASSUME( source_slot_offset < src_leaf->num_vals ); + // if we have reached the end of the rightmost leaf simply perform + // a bulk_append + if ( ( tgt_leaf_next_pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) + { + std::shift_left( src_leaf->keys, &src_leaf->keys[ src_leaf->num_vals ], source_slot_offset ); + src_leaf->num_vals -= source_slot_offset; + base::link( *tgt_leaf, *src_leaf ); + base::bulk_append_fill_incomplete_leaf( *src_leaf ); + auto const rightmost_parent_slot{ tgt_leaf->parent }; + auto const parent_pos { tgt_leaf->parent_child_idx }; // key idx = child idx - 1 & this is the 'next' key + base::bulk_append( src_leaf, { rightmost_parent_slot, parent_pos } ); + break; + } + + node_size_type consumed_source; + std::tie( consumed_source, tgt_leaf, tgt_leaf_next_pos ) = + merge + ( + *src_leaf, source_slot_offset, + *tgt_leaf, tgt_leaf_next_pos + ); + // TODO merge returns 'hints' for the next target position: use that + // for a faster version of find_nodes_for (that does not search from + // the beginning every time). + + // merge might have caused a relocation (by calling split_to_insert) + // TODO use iter_pos directly + p_new_keys .update_pool_ptr( this->nodes_ ); + p_new_nodes_end.update_pool_ptr( this->nodes_ ); + p_new_keys += consumed_source; + if ( source_slot != p_new_keys.pos().node ) + { + // merged leaves (their contents) were effectively copied into + // existing leaves (instead of simply linked into the tree + // structure) and now have to be returned to the free pool + // TODO: add leak detection to/for the entire bp_tree class + src_leaf = &this->leaf( source_slot ); // could be invalidated by split_to_insert in merge + base::unlink_right( *src_leaf ); + free( *src_leaf ); + } + } + + this->hdr().size_ += total_size; +} // bp_tree::insert() + +template +void bp_tree::merge( bp_tree & other ) +{ + // This function follows nearly the same logic as bulk insert (consult it + // for more comments), the main differences being: + // - no need to copy and sort the input + // - the bulk_append phase has to first copy the remaining of the source + // nodes (they are not somehow 'extractable' from the source tree) + // - care has to be taken around the fact that source leaves are coming + // from a different tree instance (i.e. from a different container) e.g. + // when resolving slots to node references. + if ( empty() ) { + return swap( other ); + } + + auto const total_size{ other.size() }; + this->reserve( total_size ); + + auto const p_new_nodes_begin{ other.ra_begin() }; + auto const p_new_nodes_end { other.ra_end () }; + + for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) + { + auto const tgt_location{ find_nodes_for( *p_new_keys ) }; + auto & tgt_leaf { tgt_location.leaf }; + auto const tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; + + auto * src_leaf { &other.leaf( p_new_keys.pos().node ) }; + auto const source_slot_offset{ p_new_keys.pos().value_offset }; + BOOST_ASSUME( source_slot_offset < src_leaf->num_vals ); + // simple bulk_append at the end of the rightmost leaf + if ( ( tgt_leaf_next_pos == tgt_leaf.num_vals ) && !tgt_leaf.right ) + { + // pre-copy the (remainder of the) source into fresh nodes in + // order to simply call bulk_append + node_slot src_copy_begin; + node_slot prev_src_copy_node; + for ( ;; ) + { + auto & src_leaf_copy{ this->template new_node() }; + if ( !src_copy_begin ) + { + src_copy_begin = slot_of( src_leaf_copy ); + this->move_keys( *src_leaf, source_slot_offset, src_leaf->num_vals, src_leaf_copy, 0 ); + src_leaf_copy.num_vals = src_leaf->num_vals - source_slot_offset; + src_leaf->num_vals = source_slot_offset; + this->link( tgt_leaf, src_leaf_copy ); + this->bulk_append_fill_incomplete_leaf( src_leaf_copy ); + } + else + { + this->move_keys( *src_leaf, 0, src_leaf->num_vals, src_leaf_copy, 0 ); + src_leaf_copy.num_vals = src_leaf->num_vals; + src_leaf->num_vals = 0; + this->link( this->leaf( prev_src_copy_node ), src_leaf_copy ); + } + BOOST_ASSUME( !src_leaf_copy.parent ); + BOOST_ASSUME( !src_leaf_copy.parent_child_idx ); + + if ( !src_leaf->right ) + break; + src_leaf = &other.right( *src_leaf ); + prev_src_copy_node = slot_of( src_leaf_copy ); + } + + this->bulk_append( &this->leaf( src_copy_begin ), { tgt_leaf.parent, tgt_leaf.parent_child_idx } ); + break; + } + + auto const consumed_source + { + std::get<0>( merge + ( + *src_leaf, source_slot_offset, + tgt_leaf, tgt_leaf_next_pos + )) + }; + + p_new_keys += consumed_source; + } // main loop + + this->hdr().size_ += total_size; +} // bp_tree::merge() + //------------------------------------------------------------------------------ } // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index 7d766a9..ffa9e49 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -75,7 +75,7 @@ void bptree_base::update_right_sibling_link( node_header const & left_node, node BOOST_ASSUME( slot_of( left_node ) == left_node_slot ); if ( left_node.right ) [[ likely ]] { - auto & right_back_left{ this->node( left_node.right ).left }; + auto & right_back_left{ right( left_node ).left }; BOOST_ASSUME( right_back_left != left_node_slot ); right_back_left = left_node_slot; } @@ -89,6 +89,33 @@ void bptree_base::unlink_node( node_header & node, node_header & cached_left_sib BOOST_ASSUME( right.left == slot_of( left ) ); left.right = right.right; update_right_sibling_link( left, right.left ); + free( node ); +} + +void bptree_base::unlink_left( node_header & nd ) noexcept +{ + if ( !nd.left ) + return; + auto & left_link{ left( nd ).right }; + BOOST_ASSUME( left_link == slot_of( nd ) ); + left_link = nd.left = {}; +} + +void bptree_base::unlink_right( node_header & nd ) noexcept +{ + if ( !nd.right ) + return; + auto & right_link{ right( nd ).left }; + BOOST_ASSUME( right_link == slot_of( nd ) ); + right_link = nd.right = {}; +} + +void bptree_base::link( node_header & left, node_header & right ) const noexcept +{ + BOOST_ASSUME( !left .right ); + BOOST_ASSUME( !right.left ); + left .right = slot_of( right ); + right.left = slot_of( left ); } [[ gnu::noinline, gnu::sysv_abi ]] @@ -299,29 +326,15 @@ bptree_base::create_root() bptree_base::depth_t bptree_base:: leaf_level() const noexcept { BOOST_ASSUME( hdr().depth_ ); return hdr().depth_ - 1; } bool bptree_base::is_leaf_level( depth_t const level ) const noexcept { return level == leaf_level(); } -void bptree_base::free( node_header & node ) noexcept +bool bptree_base::is_my_node( node_header const & node ) const noexcept { - BOOST_ASSUME( node.num_vals == 0 ); - auto & hdr{ this->hdr() }; - auto & free_list{ hdr.free_list_ }; - auto & free_node{ static_cast( node ) }; - auto const free_node_slot{ slot_of( free_node ) }; -#ifndef NDEBUG - if ( free_node.left ) - BOOST_ASSERT( this->node( free_node.left ).right != free_node_slot ); - if ( free_node.right ) - BOOST_ASSERT( this->node( free_node.right ).left != free_node_slot ); -#endif - free_node.left = {}; - if ( free_list ) { BOOST_ASSUME( hdr.free_node_count_ ); free_node.right = free_list; this->node( free_list ).left = free_node_slot; } - else { BOOST_ASSUME( !hdr.free_node_count_ ); free_node.right = {} ; } - free_list = free_node_slot; - ++hdr.free_node_count_; + return ( &node >= &nodes_.front() ) && ( &node <= &nodes_.back() ); } bptree_base::node_slot bptree_base::slot_of( node_header const & node ) const noexcept { + BOOST_ASSUME( is_my_node( node ) ); return { static_cast( static_cast( &node ) - nodes_.data() ) }; } @@ -339,21 +352,48 @@ bptree_base::new_node() BOOST_ASSUME( !cached_node.num_vals ); BOOST_ASSUME( !cached_node.left ); free_list = cached_node.right; - cached_node.right = {}; - if ( free_list ) - node( free_list ).left = {}; + unlink_right( cached_node ); --hdr.free_node_count_; return as( cached_node ); } auto & new_node{ nodes_.emplace_back() }; - BOOST_ASSUME( new_node.num_vals == 0 ); - new_node.left = new_node.right = {}; + BOOST_ASSUME( !new_node.num_vals ); + BOOST_ASSUME( !new_node.left ); + BOOST_ASSUME( !new_node.right ); #ifndef NDEBUG hdr_ = &this->hdr(); #endif return new_node; } +void bptree_base::free( node_header & node ) noexcept +{ + auto & hdr{ this->hdr() }; + auto & free_list{ hdr.free_list_ }; + auto & free_node{ static_cast( node ) }; + auto const free_node_slot{ slot_of( free_node ) }; +#ifndef NDEBUG + if ( free_node.left ) + BOOST_ASSERT( left( free_node ).right != free_node_slot ); + if ( free_node.right ) + BOOST_ASSERT( right( free_node ).left != free_node_slot ); +#endif + // TODO (re)consider whether the calling code is expected to reset num_vals + // (and thus mark the node as 'free'/empty) and/or unlink from its siblings + // and the parent (which complicates code but possibly catches errors + // earlier) or have free perform all of the cleanup. + //BOOST_ASSUME( node.num_vals == 0 ); + // We have to touch the header (i.e. its residing cache line) anyway here + // (to update the right link) so reset/setup the whole header right now for + // the new allocation step. + static_cast( free_node ) = {}; + // update the right link + if ( free_list ) { BOOST_ASSUME( hdr.free_node_count_ ); link( free_node, this->node( free_list ) ); } + else { BOOST_ASSUME( !hdr.free_node_count_ ); } + free_list = free_node_slot; + ++hdr.free_node_count_; +} + //------------------------------------------------------------------------------ } // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/test/b+tree.cpp b/test/b+tree.cpp index f12dd65..10736ff 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -19,32 +19,37 @@ static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) { #ifdef NDEBUG - auto const test_size{ 3553735 }; + auto const test_size{ 14553735 }; #else - auto const test_size{ 158735 }; + auto const test_size{ 258735 }; #endif std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; std::mt19937 rng{ std::random_device{}() }; auto numbers{ std::ranges::to( sorted_numbers ) }; - std::shuffle( numbers.begin(), numbers.end(), rng ); + std::span const nums{ numbers }; + // leave the largest quarter of values at the end to trigger/exercise the + // bulk_append branch in insert() + std::ranges::shuffle( nums.subspan( 0, 3 * nums.size() / 4 ), rng ); + std::ranges::shuffle( nums.subspan( 3 * nums.size() / 4 ), rng ); { bp_tree bpt; bpt.map_memory(); - bpt.reserve( numbers.size() ); + bpt.reserve( nums.size() ); { - auto const nums { std::span{ numbers } }; auto const third{ nums.size() / 3 }; auto const first_third{ nums.subspan( 0 * third, third ) }; auto const second_third{ nums.subspan( 1 * third, third ) }; auto const third_third{ nums.subspan( 2 * third ) }; - bpt.insert( first_third ); - for ( auto const & n : second_third ) { bpt.insert( n ); } - bpt.insert( third_third ); + bpt.insert( first_third ); // bulk into empty + for ( auto const & n : second_third ) { bpt.insert( n ); } // single value + bpt.insert( third_third ); // bulk into non-empty } static_assert( std::forward_iterator::const_iterator> ); - EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ) ) ); + EXPECT_EQ ( std::distance( bpt. begin(), bpt. end() ), bpt.size() ); + EXPECT_EQ ( std::distance( bpt.ra_begin(), bpt.ra_end() ), bpt.size() ); + EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ), bpt.comp() ) ); EXPECT_TRUE( std::ranges::equal( sorted_numbers, bpt ) ); EXPECT_TRUE( std::ranges::equal( sorted_numbers, bpt.random_access() ) ); EXPECT_NE( bpt.find( +42 ), bpt.end() ); @@ -56,13 +61,41 @@ TEST( bp_tree, playground ) EXPECT_TRUE( std::ranges::equal( std::views::reverse( bpt.random_access() ), std::views::reverse( sorted_numbers ) ) ); { auto const ra{ bpt.random_access() }; - for ( auto n : std::views::iota( 0, 45678 ) ) // slow operation: use a smaller subset of the input + for ( auto n : std::views::iota( 0, test_size / 55 ) ) // slow operation (not really amortized constant time): use a smaller subset of the input EXPECT_EQ( ra[ n ], n ); } + // merge test: to exercise the bulk_append path leave/add extra entries + // at the end of input data, bigger than all existing values, that are + // separately shuffled (so that they remain at the end and thus trigger + // the bulk_append branch in merge()). + auto const extra_entries_for_tree_merge{ test_size / 5 }; + std::ranges::iota_view const merge_appendix{ test_size, test_size + extra_entries_for_tree_merge }; + { + auto const evenize{ std::views::transform( []( int const v ) { return v * 2; } ) }; + + auto const even_sorted_numbers{ std::ranges::iota_view{ 0, test_size / 2 } | evenize }; + auto shuffled_even_numbers{ std::ranges::to( even_sorted_numbers ) }; + std::shuffle( shuffled_even_numbers.begin(), shuffled_even_numbers.end(), rng ); + for ( auto const & n : shuffled_even_numbers ) { + bpt.erase( n ); + } + + bp_tree bpt_even; + bpt_even.map_memory(); + shuffled_even_numbers.append_range( merge_appendix ); + bpt_even.insert( shuffled_even_numbers ); + + bpt.merge( bpt_even ); + } + + EXPECT_TRUE( std::ranges::equal( std::ranges::iota_view{ 0, test_size + extra_entries_for_tree_merge }, bpt ) ); + std::shuffle( numbers.begin(), numbers.end(), rng ); for ( auto const & n : numbers ) bpt.erase( n ); + for ( auto const & n : merge_appendix ) + bpt.erase( n ); EXPECT_TRUE( bpt.empty() ); } @@ -76,7 +109,7 @@ TEST( bp_tree, playground ) static_assert( std::forward_iterator::const_iterator> ); - EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ) ) ); + EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ), bpt.comp() ) ); EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); EXPECT_NE( bpt.find( +42 ), bpt.end() ); EXPECT_EQ( bpt.find( -42 ), bpt.end() ); @@ -90,7 +123,7 @@ TEST( bp_tree, playground ) EXPECT_EQ( bpt.size(), sorted_numbers.size() - 1 ); bpt.insert( +42 ); - EXPECT_TRUE( std::ranges::is_sorted( bpt ) ); + EXPECT_TRUE( std::ranges::is_sorted( bpt, bpt.comp() ) ); EXPECT_TRUE( std::ranges::equal( std::as_const( bpt ), sorted_numbers ) ); EXPECT_NE( bpt.find( +42 ), bpt.end() ); EXPECT_EQ( bpt.find( -42 ), bpt.end() ); From cb98a1394d7f2b6ef24dbf8d97ed22dfbddb8ab6 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Thu, 17 Oct 2024 16:06:51 +0200 Subject: [PATCH 35/49] Added support/handling for insertion of already existing values (with a std compliant interface). --- include/psi/vm/containers/b+tree.hpp | 116 +++++++++++++++++++-------- test/b+tree.cpp | 43 +++++----- 2 files changed, 105 insertions(+), 54 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 7fdc295..5b0edbc 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -1151,10 +1151,10 @@ template class bptree_base_wkey::fwd_iterator : public base_iterator, - public iter_impl + public iter_impl { private: - using impl = iter_impl; + using impl = iter_impl; using base_iterator::base_iterator; @@ -1336,30 +1336,35 @@ class bp_tree auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end() }; } BOOST_NOINLINE - void insert( key_const_arg v ) + std::pair insert( key_const_arg v ) { if ( empty() ) { auto & root{ static_cast( base::create_root() ) }; BOOST_ASSUME( root.num_vals == 1 ); root.keys[ 0 ] = v; - return; + return { begin(), true }; } auto const locations{ find_nodes_for( v ) }; BOOST_ASSUME( !locations.inner ); BOOST_ASSUME( !locations.inner_offset ); - BOOST_ASSUME( !locations.leaf_offset.exact_find ); - base::insert( locations.leaf, locations.leaf_offset.pos, v, { /*insertion starts from leaves which do not have children*/ } ); + if ( locations.leaf_offset.exact_find ) [[ unlikely ]] + return { { this->nodes_, { slot_of( locations.leaf ), locations.leaf_offset.pos } }, false }; + auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, v, { /*insertion starts from leaves which do not have children*/ } ) }; ++this->hdr().size_; + return { std::prev( iterator{ this->nodes_, { insert_pos_next.node, insert_pos_next.next_insert_offset } } ), true }; } // bulk insert + // performance note: insertion of existing values into a unique bp_tree is + // supported and accounted for (the input values are skipped) but it is + // considered an 'unlikely' event and as such it is handled by sad/cold paths // TODO proper std insert interface (w/ ranges, iterators, hints...) - void insert( std::span keys ); + size_type insert( std::span keys ); - void merge( bp_tree & other ); + size_type merge( bp_tree & other ); [[ using gnu: noinline, pure, sysv_abi ]] const_iterator find( key_const_arg key ) const noexcept @@ -1375,8 +1380,8 @@ class bp_tree return this->cend(); } - BOOST_NOINLINE - void erase( key_const_arg key ) noexcept + [[ nodiscard ]] BOOST_NOINLINE + bool erase( key_const_arg key ) noexcept { auto & hdr{ base::hdr() }; auto & __restrict depth_{ hdr.depth_ }; @@ -1405,7 +1410,9 @@ class bp_tree } - BOOST_ASSUME( location.leaf_offset.exact_find ); + if ( !location.leaf_offset.exact_find ) [[ unlikely ]] + return false; + lshift_keys( leaf, leaf_key_offset ); BOOST_ASSUME( leaf.num_vals ); --leaf.num_vals; @@ -1432,6 +1439,7 @@ class bp_tree } --size_; + return true; } void swap( bp_tree & other ) noexcept { base::swap( other ); } @@ -1550,6 +1558,10 @@ class bp_tree // support merging nodes from another tree instance auto const source_slot{ base::is_my_node( source ) ? slot_of( source ) : node_slot{} }; BOOST_ASSERT( find( target, src_keys[ 0 ] ).pos == target_offset ); + if ( eq( target.keys[ target_offset ], src_keys[ 0 ] ) ) [[ unlikely ]] + { + return std::make_tuple( 0, 1, &target, target_offset ); + } auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, src_keys[ 0 ], {} ) }; auto const & src{ source_slot ? base::leaf( source_slot ) : source }; auto & tgt{ base::leaf( target_slot ) }; @@ -1568,7 +1580,7 @@ class bp_tree src.keys[ next_src_offset ] ).pos; } - return std::make_tuple( 1, &tgt, next_tgt_offset ); + return std::make_tuple( 1, 1, &tgt, next_tgt_offset ); } auto copy_size{ std::min( input_length, available_space ) }; @@ -1578,7 +1590,7 @@ class bp_tree if ( target.right ) { auto const & right_delimiter { base::leaf( target.right ).keys[ 0 ] }; - auto less_than_right_pos{ find( src_keys, copy_size, right_delimiter ).pos }; + auto const less_than_right_pos{ find( src_keys, copy_size, right_delimiter ).pos }; if ( less_than_right_pos != copy_size ) { BOOST_ASSUME( less_than_right_pos < copy_size ); @@ -1590,24 +1602,29 @@ class bp_tree } auto & tgt_size{ target.num_vals }; + node_size_type inserted_size; if ( target_offset == tgt_size ) // a simple append { std::copy_n( src_keys, copy_size, &tgt_keys[ target_offset ] ); - tgt_size += copy_size; + tgt_size += copy_size; + inserted_size = copy_size; } else { BOOST_ASSUME( copy_size + tgt_size <= leaf_node::max_values ); std::move_backward( &tgt_keys[ 0 ], &tgt_keys[ tgt_size ], &tgt_keys[ tgt_size + copy_size ] ); - tgt_size = merge_interleaved_values + auto const new_tgt_size{ merge_interleaved_values ( &src_keys[ 0 ], copy_size, &tgt_keys[ copy_size ], tgt_size, &tgt_keys[ 0 ] - ); + ) }; + inserted_size = static_cast( new_tgt_size - tgt_size ); + tgt_size = new_tgt_size; } verify( target ); - return std::make_tuple( copy_size, &target, tgt_size ); + BOOST_ASSUME( inserted_size <= copy_size ); + return std::make_tuple( inserted_size, copy_size, &target, tgt_size ); } node_size_type merge_interleaved_values ( @@ -1663,7 +1680,8 @@ class bp_tree PSI_WARNING_DISABLE_POP() template -void bp_tree::insert( std::span keys ) +bp_tree::size_type +bp_tree::insert( std::span keys ) { // https://www.sciencedirect.com/science/article/abs/pii/S0020025502002025 On batch-constructing B+-trees: algorithm and its performance // https://www.vldb.org/conf/2001/P461.pdf An Evaluation of Generic Bulk Loading Techniques @@ -1676,14 +1694,22 @@ void bp_tree::insert( std::span keys ) std::sort( p_new_nodes_begin, p_new_nodes_end, comp() ); if ( empty() ) - return base::bulk_insert_into_empty( begin_leaf, end_pos, total_size ); + { + base::bulk_insert_into_empty( begin_leaf, end_pos, total_size ); + return total_size; + } + size_type inserted{ 0 }; for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) { auto const tgt_location{ find_nodes_for( *p_new_keys ) }; - BOOST_ASSUME( !tgt_location.leaf_offset.exact_find ); - auto tgt_leaf { &tgt_location.leaf }; - auto tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; + if ( tgt_location.leaf_offset.exact_find ) [[ unlikely ]] + { + ++p_new_keys; + continue; + } + auto tgt_leaf { &tgt_location.leaf }; + auto tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; auto const [source_slot, source_slot_offset]{ p_new_keys.pos() }; auto * src_leaf{ &this->leaf( source_slot ) }; @@ -1692,18 +1718,21 @@ void bp_tree::insert( std::span keys ) // a bulk_append if ( ( tgt_leaf_next_pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) { - std::shift_left( src_leaf->keys, &src_leaf->keys[ src_leaf->num_vals ], source_slot_offset ); + auto const so_far_consumed{ static_cast( p_new_keys - p_new_nodes_begin ) }; + BOOST_ASSUME( so_far_consumed < total_size ); + std::shift_left( &src_leaf->keys[ 0 ], &src_leaf->keys[ src_leaf->num_vals ], source_slot_offset ); src_leaf->num_vals -= source_slot_offset; base::link( *tgt_leaf, *src_leaf ); - base::bulk_append_fill_incomplete_leaf( *src_leaf ); + base::bulk_append_fill_incomplete_leaf( *src_leaf ); // yes, in case src_leaf is really incomplete, the shift_left above is redundant auto const rightmost_parent_slot{ tgt_leaf->parent }; auto const parent_pos { tgt_leaf->parent_child_idx }; // key idx = child idx - 1 & this is the 'next' key base::bulk_append( src_leaf, { rightmost_parent_slot, parent_pos } ); + inserted += static_cast( total_size - so_far_consumed ); break; } - node_size_type consumed_source; - std::tie( consumed_source, tgt_leaf, tgt_leaf_next_pos ) = + node_size_type inserted_count, consumed_source; + std::tie( inserted_count, consumed_source, tgt_leaf, tgt_leaf_next_pos ) = merge ( *src_leaf, source_slot_offset, @@ -1718,6 +1747,7 @@ void bp_tree::insert( std::span keys ) p_new_keys .update_pool_ptr( this->nodes_ ); p_new_nodes_end.update_pool_ptr( this->nodes_ ); p_new_keys += consumed_source; + inserted += inserted_count; if ( source_slot != p_new_keys.pos().node ) { // merged leaves (their contents) were effectively copied into @@ -1730,11 +1760,14 @@ void bp_tree::insert( std::span keys ) } } - this->hdr().size_ += total_size; + BOOST_ASSUME( inserted <= total_size ); + this->hdr().size_ += inserted; + return inserted; } // bp_tree::insert() template -void bp_tree::merge( bp_tree & other ) +bp_tree::size_type +bp_tree::merge( bp_tree & other ) { // This function follows nearly the same logic as bulk insert (consult it // for more comments), the main differences being: @@ -1744,8 +1777,10 @@ void bp_tree::merge( bp_tree & other ) // - care has to be taken around the fact that source leaves are coming // from a different tree instance (i.e. from a different container) e.g. // when resolving slots to node references. + // TODO further deduplicate with insert if ( empty() ) { - return swap( other ); + swap( other ); + return size(); } auto const total_size{ other.size() }; @@ -1754,9 +1789,15 @@ void bp_tree::merge( bp_tree & other ) auto const p_new_nodes_begin{ other.ra_begin() }; auto const p_new_nodes_end { other.ra_end () }; + size_type inserted{ 0 }; for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) { auto const tgt_location{ find_nodes_for( *p_new_keys ) }; + if ( tgt_location.leaf_offset.exact_find ) [[ unlikely ]] + { + ++p_new_keys; + continue; + } auto & tgt_leaf { tgt_location.leaf }; auto const tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; @@ -1798,23 +1839,28 @@ void bp_tree::merge( bp_tree & other ) prev_src_copy_node = slot_of( src_leaf_copy ); } + auto const so_far_consumed{ static_cast( p_new_keys - p_new_nodes_begin ) }; + BOOST_ASSUME( so_far_consumed < total_size ); this->bulk_append( &this->leaf( src_copy_begin ), { tgt_leaf.parent, tgt_leaf.parent_child_idx } ); + inserted += static_cast( total_size - so_far_consumed ); break; } - auto const consumed_source - { - std::get<0>( merge + node_size_type inserted_count, consumed_source; + std::tie( inserted_count, consumed_source, std::ignore, std::ignore ) = + merge ( *src_leaf, source_slot_offset, tgt_leaf, tgt_leaf_next_pos - )) - }; + ); p_new_keys += consumed_source; + inserted += inserted_count; } // main loop - this->hdr().size_ += total_size; + BOOST_ASSUME( inserted <= total_size ); + this->hdr().size_ += inserted; + return inserted; } // bp_tree::merge() //------------------------------------------------------------------------------ diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 10736ff..f48a747 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -19,9 +19,9 @@ static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) { #ifdef NDEBUG - auto const test_size{ 14553735 }; + auto const test_size{ 6853735 }; #else - auto const test_size{ 258735 }; + auto const test_size{ 258735 }; #endif std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; std::mt19937 rng{ std::random_device{}() }; @@ -40,9 +40,11 @@ TEST( bp_tree, playground ) auto const first_third{ nums.subspan( 0 * third, third ) }; auto const second_third{ nums.subspan( 1 * third, third ) }; auto const third_third{ nums.subspan( 2 * third ) }; - bpt.insert( first_third ); // bulk into empty - for ( auto const & n : second_third ) { bpt.insert( n ); } // single value - bpt.insert( third_third ); // bulk into non-empty + EXPECT_EQ( bpt.insert( first_third ) , first_third.size() ); // bulk into empty + for ( auto const & n : second_third ) { EXPECT_EQ( bpt.insert( n ).second, true ); } // single value + EXPECT_EQ( bpt.insert( third_third ) , third_third.size() ); // bulk into non-empty + + EXPECT_EQ( bpt.insert( second_third ), 0 ); } static_assert( std::forward_iterator::const_iterator> ); @@ -54,9 +56,12 @@ TEST( bp_tree, playground ) EXPECT_TRUE( std::ranges::equal( sorted_numbers, bpt.random_access() ) ); EXPECT_NE( bpt.find( +42 ), bpt.end() ); EXPECT_EQ( bpt.find( -42 ), bpt.end() ); - bpt.erase( 42 ); - EXPECT_EQ( bpt.find( +42 ), bpt.end() ); - bpt.insert( 42 ); + EXPECT_TRUE ( bpt.erase( 42 ) ); + EXPECT_FALSE( bpt.erase( 42 ) ); + EXPECT_EQ ( bpt.find( +42 ), bpt.end() ); + EXPECT_TRUE ( bpt.insert( 42 ).second ); + EXPECT_FALSE( bpt.insert( 42 ).second ); + EXPECT_EQ ( *bpt.insert( 42 ).first, 42 ); EXPECT_TRUE( std::ranges::equal( bpt.random_access() , sorted_numbers ) ); EXPECT_TRUE( std::ranges::equal( std::views::reverse( bpt.random_access() ), std::views::reverse( sorted_numbers ) ) ); { @@ -78,7 +83,7 @@ TEST( bp_tree, playground ) auto shuffled_even_numbers{ std::ranges::to( even_sorted_numbers ) }; std::shuffle( shuffled_even_numbers.begin(), shuffled_even_numbers.end(), rng ); for ( auto const & n : shuffled_even_numbers ) { - bpt.erase( n ); + EXPECT_TRUE( bpt.erase( n ) ); } bp_tree bpt_even; @@ -86,16 +91,16 @@ TEST( bp_tree, playground ) shuffled_even_numbers.append_range( merge_appendix ); bpt_even.insert( shuffled_even_numbers ); - bpt.merge( bpt_even ); + EXPECT_EQ( bpt.merge( bpt_even ), bpt_even.size() ); } EXPECT_TRUE( std::ranges::equal( std::ranges::iota_view{ 0, test_size + extra_entries_for_tree_merge }, bpt ) ); std::shuffle( numbers.begin(), numbers.end(), rng ); for ( auto const & n : numbers ) - bpt.erase( n ); + EXPECT_TRUE( bpt.erase( n ) ); for ( auto const & n : merge_appendix ) - bpt.erase( n ); + EXPECT_TRUE( bpt.erase( n ) ); EXPECT_TRUE( bpt.empty() ); } @@ -105,23 +110,23 @@ TEST( bp_tree, playground ) bpt.map_file( test_file, flags::named_object_construction_policy::create_new_or_truncate_existing ); for ( auto const & n : numbers ) - bpt.insert( n ); + EXPECT_TRUE( bpt.insert( n ).second ); static_assert( std::forward_iterator::const_iterator> ); EXPECT_TRUE( std::ranges::is_sorted( std::as_const( bpt ), bpt.comp() ) ); EXPECT_TRUE( std::ranges::equal( bpt, sorted_numbers ) ); - EXPECT_NE( bpt.find( +42 ), bpt.end() ); - EXPECT_EQ( bpt.find( -42 ), bpt.end() ); - bpt.erase( 42 ); - EXPECT_EQ( bpt.find( +42 ), bpt.end() ); + EXPECT_NE ( bpt.find( +42 ), bpt.end() ); + EXPECT_EQ ( bpt.find( -42 ), bpt.end() ); + EXPECT_TRUE( bpt.erase( 42 ) ); + EXPECT_EQ ( bpt.find( +42 ), bpt.end() ); } { bp_tree bpt; bpt.map_file( test_file, flags::named_object_construction_policy::open_existing ); - EXPECT_EQ( bpt.size(), sorted_numbers.size() - 1 ); - bpt.insert( +42 ); + EXPECT_EQ ( bpt.size(), sorted_numbers.size() - 1 ); + EXPECT_TRUE( bpt.insert( +42 ).second ); EXPECT_TRUE( std::ranges::is_sorted( bpt, bpt.comp() ) ); EXPECT_TRUE( std::ranges::equal( std::as_const( bpt ), sorted_numbers ) ); From 7fdbd641027bb87cdfc3ef9a0b91c6e17984172a Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Thu, 17 Oct 2024 18:59:05 +0200 Subject: [PATCH 36/49] Added support for insertion from generic containers and iterator pairs. --- include/psi/vm/containers/b+tree.hpp | 279 ++++++++++++++++----------- test/b+tree.cpp | 2 + 2 files changed, 167 insertions(+), 114 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 5b0edbc..6747029 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -784,30 +784,64 @@ class bptree_base_wkey : public bptree_base } } - auto bulk_insert_prepare( std::span keys ) + struct bulk_copied_input { - reserve( static_cast( keys.size() ) ); - - auto & hdr{ this->hdr() }; - auto const begin { hdr.free_list_ }; + node_slot begin; + iter_pos end; + size_type size; + }; + template + bulk_copied_input + bulk_insert_prepare( std::ranges::subrange keys ) + { + auto constexpr can_preallocate{ kind == std::ranges::subrange_kind::sized }; + if constexpr ( can_preallocate ) + reserve( static_cast( keys.size() ) ); + else + reserve( 42 ); + // w/o preallocation a saved hdr reference could get invalidated + auto const begin { can_preallocate ? hdr().free_list_ : slot_of( new_node() ) }; auto leaf_slot{ begin }; - while ( !keys.empty() ) + auto p_keys{ keys.begin() }; + size_type count{ 0 }; + for ( ;; ) { leaf_node & leaf{ this->leaf( leaf_slot ) }; BOOST_ASSUME( leaf.num_vals == 0 ); - auto const size_to_copy{ static_cast( std::min( leaf.max_values, keys.size() ) ) }; - BOOST_ASSUME( size_to_copy ); - std::copy_n( keys.begin(), size_to_copy, leaf.keys ); - leaf.num_vals = size_to_copy; - keys = keys.subspan( size_to_copy ); - --hdr.free_node_count_; - if ( !keys.empty() ) - leaf_slot = leaf.right; + if constexpr ( can_preallocate ) { + auto const size_to_copy{ static_cast( std::min( leaf.max_values, keys.end() - p_keys ) ) }; + BOOST_ASSUME( size_to_copy ); + std::copy_n( p_keys, size_to_copy, leaf.keys ); + leaf.num_vals = size_to_copy; + count += size_to_copy; + p_keys += size_to_copy; + } else { + while ( ( p_keys != keys.end() ) && ( leaf.num_vals < leaf.max_values ) ) { + leaf.keys[ leaf.num_vals++ ] = *p_keys++; + } + count += leaf.num_vals; + } + --this->hdr().free_node_count_; + if ( p_keys != keys.end() ) + { + if constexpr ( can_preallocate ) { + leaf_slot = leaf.right; + } else { + auto & new_leaf{ new_node() }; + link( leaf, new_leaf ); + leaf_slot = slot_of( new_leaf ); + } + BOOST_ASSUME( !!leaf_slot ); + } else { - hdr.free_list_ = leaf.right; - unlink_right( leaf ); - return std::make_pair( begin, iter_pos{ leaf_slot, size_to_copy } ); + if constexpr ( can_preallocate ) { + this->hdr().free_list_ = leaf.right; + unlink_right( leaf ); + BOOST_ASSUME( count == static_cast( keys.size() ) ); + count = static_cast( keys.size() ); // eliminate the accumulation code above + } + return bulk_copied_input{ begin, { leaf_slot, leaf.num_vals }, count }; } } std::unreachable(); @@ -1314,6 +1348,9 @@ class bp_tree using base::size; using base::clear; + bp_tree() noexcept = default; + bp_tree( Comparator const & comp ) noexcept : Comparator{ comp } {} + static constexpr base::size_type max_size() noexcept { auto const max_number_of_nodes { std::numeric_limits::max() }; @@ -1332,8 +1369,8 @@ class bp_tree [[ gnu::pure ]] std::basic_const_iterator ra_begin() const noexcept { return const_cast( *this ).ra_begin(); } [[ gnu::pure ]] std::basic_const_iterator ra_end () const noexcept { return const_cast( *this ).ra_end (); } - auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end() }; } - auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end() }; } + auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } + auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } BOOST_NOINLINE std::pair insert( key_const_arg v ) @@ -1361,8 +1398,10 @@ class bp_tree // performance note: insertion of existing values into a unique bp_tree is // supported and accounted for (the input values are skipped) but it is // considered an 'unlikely' event and as such it is handled by sad/cold paths - // TODO proper std insert interface (w/ ranges, iterators, hints...) - size_type insert( std::span keys ); + // TODO complete std insert interface (w/ ranges, iterators, hints...) + template + size_type insert( InIter const begin, InIter const end ) { return insert( base::bulk_insert_prepare( std::ranges::subrange( begin, end ) ) ); } + size_type insert( std::ranges::range auto const & keys ) { return insert( base::bulk_insert_prepare( std::ranges::subrange( keys ) ) ); } size_type merge( bp_tree & other ); @@ -1532,100 +1571,15 @@ class bp_tree }; } + size_type insert( typename base::bulk_copied_input ); + // bulk insert helper: merge a new, presorted leaf into an existing leaf auto merge ( - leaf_node const & source, node_size_type const source_offset, - leaf_node & target, node_size_type const target_offset - ) noexcept - { - verify( source ); - verify( target ); - BOOST_ASSUME( source_offset < source.num_vals ); - node_size_type input_length ( source.num_vals - source_offset ); - node_size_type const available_space( target.max_values - target.num_vals ); - auto const src_keys{ &source.keys[ source_offset ] }; - auto & tgt_keys{ target.keys }; - if ( target_offset == 0 ) [[ unlikely ]] - { - auto const & new_separator{ src_keys[ 0 ] }; - BOOST_ASSUME( new_separator < tgt_keys[ 0 ] ); - base::update_separator( target, new_separator ); - // TODO rather simply insert the source leaf into the parent - } - if ( !available_space ) [[ unlikely ]] - { - // support merging nodes from another tree instance - auto const source_slot{ base::is_my_node( source ) ? slot_of( source ) : node_slot{} }; - BOOST_ASSERT( find( target, src_keys[ 0 ] ).pos == target_offset ); - if ( eq( target.keys[ target_offset ], src_keys[ 0 ] ) ) [[ unlikely ]] - { - return std::make_tuple( 0, 1, &target, target_offset ); - } - auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, src_keys[ 0 ], {} ) }; - auto const & src{ source_slot ? base::leaf( source_slot ) : source }; - auto & tgt{ base::leaf( target_slot ) }; - BOOST_ASSUME( next_tgt_offset <= tgt.num_vals ); - auto const next_src_offset{ static_cast( source_offset + 1 ) }; - // next_tgt_offset returned by split_to_insert points to the - // position in the target node that immediately follows the - // position for the inserted src_keys[ 0 ] - IOW it need not be - // the position for src.keys[ next_src_offset ] - if ( tgt.num_vals != next_tgt_offset ) // necessary check because find assumes non-empty input - { - next_tgt_offset += find - ( - &tgt.keys[ next_tgt_offset ], - tgt.num_vals - next_tgt_offset, - src.keys[ next_src_offset ] - ).pos; - } - return std::make_tuple( 1, 1, &tgt, next_tgt_offset ); - } - - auto copy_size{ std::min( input_length, available_space ) }; - // If there is an existing right sibling we must first check if the - // source contains values beyond its separator key (and adjust the copy - // size accordingly to maintain the sorted property). - if ( target.right ) - { - auto const & right_delimiter { base::leaf( target.right ).keys[ 0 ] }; - auto const less_than_right_pos{ find( src_keys, copy_size, right_delimiter ).pos }; - if ( less_than_right_pos != copy_size ) - { - BOOST_ASSUME( less_than_right_pos < copy_size ); - node_size_type const input_end_for_target( less_than_right_pos + source_offset ); - BOOST_ASSUME( input_end_for_target > source_offset ); - BOOST_ASSUME( input_end_for_target <= source.num_vals ); - copy_size = static_cast( input_end_for_target - source_offset ); - } - } + leaf_node const & source, node_size_type source_offset, + leaf_node & target, node_size_type target_offset + ) noexcept; - auto & tgt_size{ target.num_vals }; - node_size_type inserted_size; - if ( target_offset == tgt_size ) // a simple append - { - std::copy_n( src_keys, copy_size, &tgt_keys[ target_offset ] ); - tgt_size += copy_size; - inserted_size = copy_size; - } - else - { - BOOST_ASSUME( copy_size + tgt_size <= leaf_node::max_values ); - std::move_backward( &tgt_keys[ 0 ], &tgt_keys[ tgt_size ], &tgt_keys[ tgt_size + copy_size ] ); - auto const new_tgt_size{ merge_interleaved_values - ( - &src_keys[ 0 ], copy_size, - &tgt_keys[ copy_size ], tgt_size, - &tgt_keys[ 0 ] - ) }; - inserted_size = static_cast( new_tgt_size - tgt_size ); - tgt_size = new_tgt_size; - } - verify( target ); - BOOST_ASSUME( inserted_size <= copy_size ); - return std::make_tuple( inserted_size, copy_size, &target, tgt_size ); - } node_size_type merge_interleaved_values ( Key const source0[], node_size_type const source0_size, @@ -1679,16 +1633,113 @@ class bp_tree PSI_WARNING_DISABLE_POP() + +template +// bulk insert helper: merge a new, presorted leaf into an existing leaf +auto bp_tree::merge +( + leaf_node const & source, node_size_type const source_offset, + leaf_node & target, node_size_type const target_offset +) noexcept +{ + verify( source ); + verify( target ); + BOOST_ASSUME( source_offset < source.num_vals ); + node_size_type input_length ( source.num_vals - source_offset ); + node_size_type const available_space( target.max_values - target.num_vals ); + auto const src_keys{ &source.keys[ source_offset ] }; + auto & tgt_keys{ target.keys }; + if ( target_offset == 0 ) [[ unlikely ]] + { + auto const & new_separator{ src_keys[ 0 ] }; + BOOST_ASSUME( new_separator < tgt_keys[ 0 ] ); + base::update_separator( target, new_separator ); + // TODO rather simply insert the source leaf into the parent + } + if ( !available_space ) [[ unlikely ]] + { + // support merging nodes from another tree instance + auto const source_slot{ base::is_my_node( source ) ? slot_of( source ) : node_slot{} }; + BOOST_ASSERT( find( target, src_keys[ 0 ] ).pos == target_offset ); + if ( eq( target.keys[ target_offset ], src_keys[ 0 ] ) ) [[ unlikely ]] + { + return std::make_tuple( 0, 1, &target, target_offset ); + } + auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, src_keys[ 0 ], {} ) }; + auto const & src{ source_slot ? base::leaf( source_slot ) : source }; + auto & tgt{ base::leaf( target_slot ) }; + BOOST_ASSUME( next_tgt_offset <= tgt.num_vals ); + auto const next_src_offset{ static_cast( source_offset + 1 ) }; + // next_tgt_offset returned by split_to_insert points to the + // position in the target node that immediately follows the + // position for the inserted src_keys[ 0 ] - IOW it need not be + // the position for src.keys[ next_src_offset ] + if ( tgt.num_vals != next_tgt_offset ) // necessary check because find assumes non-empty input + { + next_tgt_offset += find + ( + &tgt.keys[ next_tgt_offset ], + tgt.num_vals - next_tgt_offset, + src.keys[ next_src_offset ] + ).pos; + } + return std::make_tuple( 1, 1, &tgt, next_tgt_offset ); + } + + auto copy_size{ std::min( input_length, available_space ) }; + // If there is an existing right sibling we must first check if the + // source contains values beyond its separator key (and adjust the copy + // size accordingly to maintain the sorted property). + if ( target.right ) + { + auto const & right_delimiter { base::leaf( target.right ).keys[ 0 ] }; + auto const less_than_right_pos{ find( src_keys, copy_size, right_delimiter ).pos }; + if ( less_than_right_pos != copy_size ) + { + BOOST_ASSUME( less_than_right_pos < copy_size ); + node_size_type const input_end_for_target( less_than_right_pos + source_offset ); + BOOST_ASSUME( input_end_for_target > source_offset ); + BOOST_ASSUME( input_end_for_target <= source.num_vals ); + copy_size = static_cast( input_end_for_target - source_offset ); + } + } + + auto & tgt_size{ target.num_vals }; + node_size_type inserted_size; + if ( target_offset == tgt_size ) // a simple append + { + std::copy_n( src_keys, copy_size, &tgt_keys[ target_offset ] ); + tgt_size += copy_size; + inserted_size = copy_size; + } + else + { + BOOST_ASSUME( copy_size + tgt_size <= leaf_node::max_values ); + std::move_backward( &tgt_keys[ 0 ], &tgt_keys[ tgt_size ], &tgt_keys[ tgt_size + copy_size ] ); + auto const new_tgt_size{ merge_interleaved_values + ( + &src_keys[ 0 ], copy_size, + &tgt_keys[ copy_size ], tgt_size, + &tgt_keys[ 0 ] + ) }; + inserted_size = static_cast( new_tgt_size - tgt_size ); + tgt_size = new_tgt_size; + } + verify( target ); + BOOST_ASSUME( inserted_size <= copy_size ); + return std::make_tuple( inserted_size, copy_size, &target, tgt_size ); +} + + template bp_tree::size_type -bp_tree::insert( std::span keys ) +bp_tree::insert( typename base::bulk_copied_input const input ) { // https://www.sciencedirect.com/science/article/abs/pii/S0020025502002025 On batch-constructing B+-trees: algorithm and its performance // https://www.vldb.org/conf/2001/P461.pdf An Evaluation of Generic Bulk Loading Techniques // https://stackoverflow.com/questions/15996319/is-there-any-algorithm-for-bulk-loading-in-b-tree - auto const total_size{ static_cast( keys.size() ) }; - auto const [begin_leaf, end_pos]{ base::bulk_insert_prepare( keys ) }; + auto const [begin_leaf, end_pos, total_size]{ input }; ra_iterator const p_new_nodes_begin{ *this, { begin_leaf, 0 }, 0 }; ra_iterator const p_new_nodes_end { *this, end_pos , total_size }; std::sort( p_new_nodes_begin, p_new_nodes_end, comp() ); diff --git a/test/b+tree.cpp b/test/b+tree.cpp index f48a747..f3d2189 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -18,6 +18,8 @@ static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) { + // TODO different types, insertion from non contiguous containers + #ifdef NDEBUG auto const test_size{ 6853735 }; #else From 47d93ab61c020d66a51d382f1ade9f6ac6f7e5b6 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Mon, 21 Oct 2024 16:52:48 +0200 Subject: [PATCH 37/49] Minor stylistic cleanup. --- include/psi/vm/align.hpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/include/psi/vm/align.hpp b/include/psi/vm/align.hpp index 2d185b2..ca372c8 100644 --- a/include/psi/vm/align.hpp +++ b/include/psi/vm/align.hpp @@ -18,7 +18,7 @@ namespace align_detail { [[ gnu::const ]] constexpr auto generic_divide_up( auto const numerator, auto const denominator ) noexcept { - return static_cast< decltype( numerator ) >( ( numerator + denominator - 1 ) / denominator ); + return static_cast( ( numerator + denominator - 1 ) / denominator ); } } // namespace detail @@ -36,10 +36,10 @@ namespace align_detail return __builtin_align_down( value, alignment ); else #endif - if constexpr ( std::is_pointer_v< T > ) - return std::bit_cast< T >( align_down( std::bit_cast< std::uintptr_t >( value ), alignment ) ); + if constexpr ( std::is_pointer_v ) + return std::bit_cast( align_down( std::bit_cast( value ), alignment ) ); else - return static_cast< T >( value / alignment * alignment ); + return static_cast( value / alignment * alignment ); } [[ using gnu: const, always_inline ]] constexpr auto align_up( auto const value, auto const alignment ) noexcept { @@ -49,21 +49,21 @@ namespace align_detail return __builtin_align_up( value, alignment ); else #endif - if constexpr ( std::is_pointer_v< T > ) - return std::bit_cast< T >( align_up( std::bit_cast< std::uintptr_t >( value ), alignment ) ); + if constexpr ( std::is_pointer_v ) + return std::bit_cast( align_up( std::bit_cast( value ), alignment ) ); else - return static_cast< T >( align_detail::generic_divide_up( value, alignment ) * alignment ); + return static_cast( align_detail::generic_divide_up( value, alignment ) * alignment ); } -template < unsigned alignment > [[ using gnu: const, always_inline ]] constexpr auto align_down( auto const value ) noexcept { return align_down( value, alignment ); } -template < unsigned alignment > [[ using gnu: const, always_inline ]] constexpr auto align_up ( auto const value ) noexcept { return align_up ( value, alignment ); } +template [[ using gnu: const, always_inline ]] constexpr auto align_down( auto const value ) noexcept { return align_down( value, alignment ); } +template [[ using gnu: const, always_inline ]] constexpr auto align_up ( auto const value ) noexcept { return align_up ( value, alignment ); } [[ using gnu: const, always_inline ]] constexpr auto divide_up( auto const numerator, auto const denominator ) noexcept { #ifdef __GNUC__ if ( __builtin_constant_p( denominator ) && /*is power of 2*/std::has_single_bit( unsigned( denominator ) ) ) - return static_cast< decltype( numerator ) >( align_up( numerator, denominator ) / denominator ); + return static_cast( align_up( numerator, denominator ) / denominator ); else #endif // GCC&co. return align_detail::generic_divide_up( numerator, denominator ); From daccd8b24f26229f65ca6e0282bbc2fb56bbbe31 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Tue, 22 Oct 2024 10:42:07 +0200 Subject: [PATCH 38/49] Support for generic lookup/transparent comparators (with logic for automatic optimal ABI). Support for bulk insert from non-contiguous containers. Added the find_next() helper function to speedup bulk operations (and in the future enable hinted-insertion like operations). Added lower_bound() method. Separate reserve and reserve_additional methods. Minor work on supporting non-trivial types. --- include/psi/vm/containers/b+tree.hpp | 532 +++++++++++++++++++-------- src/containers/b+tree.cpp | 30 +- 2 files changed, 405 insertions(+), 157 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 6747029..742e06b 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -45,9 +45,92 @@ namespace detail template static constexpr bool is_simple_comparator>{ true }; } // namespace detail +//////////////////////////////////////////////////////////////////////////////// +// Modern(ized) attempt at 'automatized' boost::call_traits primarily to support +// efficient transparent comparators & non-inlined generic lookup functions +// which cause neither unnecessary copies of non-trivial types nor pass-by-ref +// of trivial ones. +// Largely still WiP... +// Essentially this is 'explicit IPA SROA'. +// https://gcc.gnu.org/onlinedocs/gccint/passes-and-files-of-the-compiler/inter-procedural-optimization-passes.html +//////////////////////////////////////////////////////////////////////////////// + +template +bool constexpr can_be_passed_in_reg +{ + ( + std::is_trivial_v && + ( sizeof( T ) <= 2 * sizeof( void * ) ) // assuming a sane ABI like SysV (ignoring the MS x64 disaster) + ) +#if defined( __GNUC__ ) || defined( __clang__ ) + || // detect SIMD types (this could also produce false positives for large compiler-native vectors that do not fit into the register file) + requires{ __builtin_convertvector( T{}, T ); } +#endif + // This is certainly not an exhaustive list/'trait' - certain types that can + // be passed in reg cannot be detected as such by existing compiler + // functionality, e.g. Homogeneous Vector Aggregates + // https://devblogs.microsoft.com/cppblog/introducing-vector-calling-convention + // users are encouraged to provide specializations for such types. +}; // can_be_passed_in_reg + +template +struct optimal_const_ref { using type = T const &; }; + +template +struct optimal_const_ref> { using type = std::basic_string_view; }; + +template +struct optimal_const_ref { using type = std::span const>; }; + +template +struct [[ clang::trivial_abi ]] pass_in_reg +{ + static auto constexpr pass_by_val{ can_be_passed_in_reg }; + + using value_type = T; + using stored_type = std::conditional_t>; + + constexpr pass_in_reg( T const & u ) noexcept : val{ u } {} + + stored_type val; + + [[ gnu::pure ]] BOOST_FORCEINLINE + constexpr operator stored_type const &() const noexcept { return val; } +}; // pass_in_reg + +template +struct [[ clang::trivial_abi ]] pass_rv_in_reg +{ + static auto constexpr pass_by_val{ can_be_passed_in_reg }; + using value_type = T; + using stored_type = std::conditional_t; + + constexpr pass_rv_in_reg( T && u ) noexcept : val{ std::move( u ) } {} // move for not-trivially-moveable yet trivial_abi types (that can be passed in reg) + + stored_type val; + + [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type const & () const noexcept { return val ; } + [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type &&() noexcept { return std::move( val ); } +}; // pass_rv_in_reg + +template +concept KeyType = transparent_comparator || std::is_same_v; + +template bool constexpr reg { false }; +template bool constexpr reg>{ true }; +template bool constexpr reg>{ true }; + +template +concept Reg = reg; + +// 'Explicit IPA SROA' / pass-in-reg helper end +//////////////////////////////////////////////////////////////////////////////// + + +// user specializations and overloads of this function are allowed template -constexpr bool use_linear_search_for_sorted_array( [[ maybe_unused ]] std::uint32_t const minimum_array_length, std::uint32_t const maximum_array_length ) noexcept +consteval bool use_linear_search_for_sorted_array( [[ maybe_unused ]] std::uint32_t const minimum_array_length, std::uint32_t const maximum_array_length ) noexcept { auto const basic_test { @@ -92,18 +175,7 @@ class bptree_base hdr() = {}; return success; } - storage_result map_memory( std::uint32_t const initial_capacity_as_number_of_nodes = 0 ) noexcept - { - storage_result success{ nodes_.map_memory( initial_capacity_as_number_of_nodes ) }; - if ( std::move( success ) ) - { - hdr() = {}; - if ( initial_capacity_as_number_of_nodes ) { - assign_nodes_to_free_pool( 0 ); - } - } - return success; - } + storage_result map_memory( std::uint32_t initial_capacity_as_number_of_nodes = 0 ) noexcept; protected: static constexpr auto node_size{ page_size }; @@ -211,7 +283,8 @@ class bptree_base void free( node_header & ) noexcept; - void reserve( node_slot::value_type additional_nodes ); + void reserve_additional( node_slot::value_type additional_nodes ); + void reserve ( node_slot::value_type new_capacity_in_number_of_nodes ); [[ gnu::pure ]] header & hdr() noexcept; [[ gnu::pure ]] header const & hdr() const noexcept { return const_cast( *this ).hdr(); } @@ -225,10 +298,10 @@ class bptree_base //BOOST_ASSUME( node.num_vals >= node.min_values ); } - static constexpr auto keys ( auto & node ) noexcept { return std::span{ node.keys , node.num_vals }; } - static constexpr auto keys ( auto const & node ) noexcept { return std::span{ node.keys , node.num_vals }; } - static constexpr auto children( auto & node ) noexcept { if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } - static constexpr auto children( auto const & node ) noexcept { if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } + static constexpr auto keys ( auto & node ) noexcept { verify( node ); return std::span{ node.keys , node.num_vals }; } + static constexpr auto keys ( auto const & node ) noexcept { verify( node ); return std::span{ node.keys , node.num_vals }; } + static constexpr auto children( auto & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } + static constexpr auto children( auto const & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } [[ gnu::pure ]] static constexpr node_size_type num_vals ( auto const & node ) noexcept { return node.num_vals; } [[ gnu::pure ]] static constexpr node_size_type num_chldrn( auto const & node ) noexcept { if constexpr ( requires{ node.children; } ) { BOOST_ASSUME( node.num_vals ); return node.num_vals + 1U; } else return 0; } @@ -434,15 +507,42 @@ class bptree_base_wkey : public bptree_base using key_type = Key; using value_type = key_type; // TODO map support - using key_const_arg = std::conditional_t, Key, Key const &>; + // support for non trivial types (for which move and pass-by-ref matters) is WiP, nowhere near complete + static_assert( std::is_trivial_v ); + + using key_rv_arg = std::conditional_t, Key const, pass_rv_in_reg>; + using key_const_arg = std::conditional_t, Key const, pass_in_reg >; public: - void reserve( size_type additional_values ) + storage_result map_memory( size_type initial_capacity = 0 ) noexcept { return bptree_base::map_memory( node_count_required_for_values( initial_capacity ) ); } + size_type capacity() const noexcept { - additional_values = additional_values * 3 / 2; // TODO find the appropriate formula - bptree_base::reserve( static_cast( ( additional_values + leaf_node::max_values - 1 ) / leaf_node::max_values ) ); + // TODO WiP playground + auto const n{ nodes_.capacity() }; + if ( !n ) [[ unlikely ]] + return 0; + + node_slot::value_type inner_nodes{ 0 }; + node_slot::value_type current_level_count{ 1 }; + while ( ( current_level_count * inner_node::max_children ) < ( n - inner_nodes ) ) + { + inner_nodes += current_level_count; + current_level_count *= inner_node::max_children; + } + + std::uint8_t const depth{ hdr().depth_ }; + std::uint8_t max_inner_node_count{ depth > 1 }; + for ( auto d{ 3 }; d < depth; ++d ) + { + max_inner_node_count += max_inner_node_count * inner_node::max_children; + } + BOOST_ASSUME( max_inner_node_count < n ); + return ( n - max_inner_node_count ) * leaf_node::max_values; } + void reserve_additional( size_type const additional_values ) { bptree_base::reserve_additional( node_count_required_for_values( additional_values ) ); } + void reserve ( size_type const new_capacity ) { bptree_base::reserve ( node_count_required_for_values( new_capacity ) ); }; + // solely a debugging helper (include b+tree_print.hpp) void print() const; @@ -515,18 +615,19 @@ class bptree_base_wkey : public bptree_base class ra_iterator; protected: // split_to_insert and its helpers - void new_root( node_slot const left_child, node_slot const right_child, key_const_arg separator_key ) + root_node & new_root( node_slot const left_child, node_slot const right_child, key_rv_arg separator_key ) { auto & new_root_node{ as( bptree_base::new_root( left_child, right_child ) ) }; new_root_node.keys [ 0 ] = std::move( separator_key ); new_root_node.children[ 0 ] = left_child; new_root_node.children[ 1 ] = right_child; + return new_root_node; } auto insert_into_new_node ( inner_node & node, inner_node & new_node, - key_const_arg value, + key_rv_arg value, node_size_type const insert_pos, node_size_type const new_insert_pos, node_slot const key_right_child ) noexcept @@ -562,7 +663,7 @@ class bptree_base_wkey : public bptree_base move_keys ( node, insert_pos , node.num_vals , new_node, new_insert_pos ); move_chldrn( node, insert_pos + 1, node.num_vals + 1, new_node, new_insert_pos + 1 ); - keys( new_node )[ new_insert_pos - 1 ] = value; + keys( new_node )[ new_insert_pos - 1 ] = std::move( value ); } insrt_child( new_node, new_insert_pos, key_right_child ); @@ -577,7 +678,7 @@ class bptree_base_wkey : public bptree_base static auto insert_into_new_node ( leaf_node & node, leaf_node & new_node, - key_const_arg value, + key_rv_arg value, node_size_type const insert_pos, node_size_type const new_insert_pos, node_slot const key_right_child ) noexcept @@ -598,7 +699,7 @@ class bptree_base_wkey : public bptree_base node .num_vals = mid ; new_node.num_vals = max - mid + 1; - keys( new_node )[ new_insert_pos ] = value; + keys( new_node )[ new_insert_pos ] = std::move( value ); auto const & key_to_propagate{ new_node.keys[ 0 ] }; BOOST_ASSUME( !underflowed( node ) ); @@ -607,7 +708,7 @@ class bptree_base_wkey : public bptree_base return std::make_pair( key_to_propagate, static_cast( new_insert_pos + 1 ) ); } - auto insert_into_existing_node( inner_node & node, inner_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + auto insert_into_existing_node( inner_node & node, inner_node & new_node, key_rv_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept { BOOST_ASSUME( bool( key_right_child ) ); @@ -630,16 +731,16 @@ class bptree_base_wkey : public bptree_base node .num_vals = mid; new_node.num_vals = max - mid; - keys ( node )[ insert_pos ] = value; + keys ( node )[ insert_pos ] = std::move( value ); insrt_child( node, insert_pos + 1, key_right_child ); BOOST_ASSUME( !underflowed( node ) ); BOOST_ASSUME( !underflowed( new_node ) ); - return std::make_pair( key_to_propagate, static_cast( insert_pos + 1 ) ); + return std::make_pair( std::move( key_to_propagate ), static_cast( insert_pos + 1 ) ); } - static auto insert_into_existing_node( leaf_node & node, leaf_node & new_node, key_const_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept + static auto insert_into_existing_node( leaf_node & node, leaf_node & new_node, key_rv_arg value, node_size_type const insert_pos, node_slot const key_right_child ) noexcept { BOOST_ASSUME( !key_right_child ); @@ -657,7 +758,7 @@ class bptree_base_wkey : public bptree_base node .num_vals = mid; new_node.num_vals = max - mid + 1; - keys( node )[ insert_pos ] = value; + keys( node )[ insert_pos ] = std::move( value ); auto const & key_to_propagate{ new_node.keys[ 0 ] }; BOOST_ASSUME( !underflowed( node ) ); @@ -667,7 +768,7 @@ class bptree_base_wkey : public bptree_base } template - insert_pos_t split_to_insert( N & node_to_split, node_size_type const insert_pos, key_const_arg value, node_slot const key_right_child ) + insert_pos_t split_to_insert( N & node_to_split, node_size_type const insert_pos, key_rv_arg value, node_slot const key_right_child ) { auto const max{ N::max_values }; auto const mid{ N::min_values }; @@ -685,8 +786,8 @@ class bptree_base_wkey : public bptree_base auto const new_insert_pos { insert_pos - mid }; bool const insertion_into_new_node{ new_insert_pos >= 0 }; auto [key_to_propagate, next_insert_pos]{ insertion_into_new_node // we cannot save a reference here because it might get invalidated by the new_node() call below - ? insert_into_new_node ( *p_node, *p_new_node, value, insert_pos, static_cast( new_insert_pos ), key_right_child ) - : insert_into_existing_node( *p_node, *p_new_node, value, insert_pos, key_right_child ) + ? insert_into_new_node ( *p_node, *p_new_node, std::move( value ), insert_pos, static_cast( new_insert_pos ), key_right_child ) + : insert_into_existing_node( *p_node, *p_new_node, std::move( value ), insert_pos, key_right_child ) }; verify( *p_node ); @@ -712,15 +813,15 @@ class bptree_base_wkey : public bptree_base protected: // 'other' template - insert_pos_t insert( N & target_node, node_size_type const target_node_pos, key_const_arg v, node_slot const right_child ) + insert_pos_t insert( N & target_node, node_size_type const target_node_pos, key_rv_arg v, node_slot const right_child ) { verify( target_node ); if ( full( target_node ) ) [[ unlikely ]] { - return split_to_insert( target_node, target_node_pos, v, right_child ); + return split_to_insert( target_node, target_node_pos, std::move( v ), right_child ); } else { ++target_node.num_vals; rshift_keys( target_node, target_node_pos ); - target_node.keys[ target_node_pos ] = v; + target_node.keys[ target_node_pos ] = std::move( v ); if constexpr ( requires { target_node.children; } ) { node_size_type const ch_pos( target_node_pos + /*>right< child*/ 1 ); rshift_chldrn( target_node, ch_pos ); @@ -763,7 +864,7 @@ class bptree_base_wkey : public bptree_base ( rightmost_parent, rightmost_parent_pos.next_insert_offset, - src_leaf->keys[ 0 ], + key_rv_arg{ auto{ src_leaf->keys[ 0 ] } }, slot_of( *src_leaf ) ); if ( !next_src_slot ) @@ -796,9 +897,9 @@ class bptree_base_wkey : public bptree_base { auto constexpr can_preallocate{ kind == std::ranges::subrange_kind::sized }; if constexpr ( can_preallocate ) - reserve( static_cast( keys.size() ) ); + reserve_additional( static_cast( keys.size() ) ); else - reserve( 42 ); + reserve_additional( 42 ); // w/o preallocation a saved hdr reference could get invalidated auto const begin { can_preallocate ? hdr().free_list_ : slot_of( new_node() ) }; auto leaf_slot{ begin }; @@ -838,7 +939,7 @@ class bptree_base_wkey : public bptree_base if constexpr ( can_preallocate ) { this->hdr().free_list_ = leaf.right; unlink_right( leaf ); - BOOST_ASSUME( count == static_cast( keys.size() ) ); + BOOST_ASSERT( count == static_cast( keys.size() ) ); count = static_cast( keys.size() ); // eliminate the accumulation code above } return bulk_copied_input{ begin, { leaf_slot, leaf.num_vals }, count }; @@ -850,29 +951,34 @@ class bptree_base_wkey : public bptree_base void bulk_insert_into_empty( node_slot const begin_leaf, iter_pos const end_leaf, size_type const total_size ) { BOOST_ASSUME( empty() ); - auto & hdr{ this->hdr() }; - hdr.root_ = begin_leaf; - hdr.first_leaf_ = begin_leaf; + auto * hdr{ &this->hdr() }; + hdr->root_ = begin_leaf; + hdr->first_leaf_ = begin_leaf; if ( begin_leaf == end_leaf.node ) [[ unlikely ]] // single-node-sized initial insert { - hdr.last_leaf_ = end_leaf.node; + hdr->last_leaf_ = end_leaf.node; return; } - auto const & first_root_left { leaf( begin_leaf ) }; - auto & first_root_right{ leaf( first_root_left.right ) }; + auto const & first_root_left { leaf ( begin_leaf ) }; + auto & first_root_right{ right( first_root_left ) }; first_root_right.parent_child_idx = 1; - hdr.depth_ = 1; - new_root( begin_leaf, first_root_left.right, first_root_right.keys[ 0 ] ); // may invalidate the hdr reference - BOOST_ASSUME( this->hdr().depth_ == 2 ); - bulk_append( &leaf( first_root_right.right ), { hdr.root_, 1 } ); - BOOST_ASSUME( this->hdr().last_leaf_ == end_leaf.node ); - this->hdr().size_ = total_size; + hdr->depth_ = 1; + auto const first_unconnected_node{ first_root_right.right }; + new_root( begin_leaf, first_root_left.right, key_rv_arg{ auto{ first_root_right.keys[ 0 ] } } ); // may invalidate references + hdr = &this->hdr(); + BOOST_ASSUME( hdr->depth_ == 2 ); + bulk_append( &leaf( first_unconnected_node ), { hdr->root_, 1 } ); + BOOST_ASSUME( hdr->last_leaf_ == end_leaf.node ); + hdr->size_ = total_size; } leaf_node & leaf ( node_slot const slot ) noexcept { return node< leaf_node>( slot ); } inner_node & inner ( node_slot const slot ) noexcept { return node( slot ); } inner_node & parent( node_header & child ) noexcept { return inner( child.parent ); } + leaf_node const & leaf ( node_slot const slot ) const noexcept { return const_cast( *this ).leaf( slot ); } + inner_node const & parent( node_header const & child ) const noexcept { return const_cast( *this ).parent( const_cast( child ) ); } + void update_separator( leaf_node & leaf, Key const & new_separator ) noexcept { // the leftmost leaf does not have a separator key (at all) @@ -903,30 +1009,6 @@ class bptree_base_wkey : public bptree_base parent_key = new_separator; } - void move( leaf_node & source, leaf_node & target ) noexcept - { - BOOST_ASSUME( &source != &target ); - BOOST_ASSUME( source.num_values >= leaf_node::min_vals ); - BOOST_ASSUME( source.num_values <= leaf_node::max_vals ); - - auto constexpr copy_whole{ std::is_trivial_v && false }; - if constexpr ( copy_whole ) - { - std::memcpy( &target, &source, sizeof( target ) ); - } - else - if constexpr ( std::is_trivial_v ) - { - std::memcpy( &target, &source, reinterpret_cast( &source.keys[ source.num_vals ] ) - reinterpret_cast( &source ) ); - } - else - { - std::ranges::uninitialized_move( keys( source ), target.keys ); - static_cast( target ) = std::move( source ); - source.num_vals = 0; - } - } - template BOOST_NOINLINE void handle_underflow( N & node, depth_t const level ) noexcept @@ -1175,6 +1257,30 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( parent.num_vals ); parent.num_vals--; } + +private: + [[ gnu::const, gnu::noinline ]] + static node_slot::value_type node_count_required_for_values( size_type const number_of_values ) noexcept + { + if ( !number_of_values ) + return 0; + auto const leaf_count{ static_cast( divide_up( number_of_values, /*assuming an 'optimistic' reserve, i.e. for bulk insert*/leaf_node::max_values ) ) }; + auto total_count{ node_slot::value_type{ 0 } }; + auto current_level_count{ leaf_count }; + auto depth{ 1 }; + while ( current_level_count > 1 ) + { + total_count += current_level_count; + current_level_count = divide_up( current_level_count, inner_node::min_children ); // pessimistic about inner node utilization + ++depth; + } + // theoretical (+1 since we use a 1-based depth index) + auto const minimum_height{ static_cast( 1 + std::ceil( std::log( number_of_values + 1 ) / std::log( inner_node::max_children ) ) - 1 ) }; + auto const maximum_height{ static_cast( 1 + std::log( ( number_of_values + 1 ) / 2 ) / std::log( inner_node::min_children ) ) }; + BOOST_ASSUME( depth >= minimum_height ); + BOOST_ASSUME( depth <= maximum_height ); + return total_count; + } }; // class bptree_base_wkey //////////////////////////////////////////////////////////////////////////////// @@ -1311,11 +1417,11 @@ class bp_tree using parent_node = base::parent_node; using fwd_iterator = base::fwd_iterator; using ra_iterator = base:: ra_iterator; + using iter_pos = base::iter_pos; using bptree_base::as; using bptree_base::can_borrow; using bptree_base::children; - using bptree_base::ihalf_ceil; using bptree_base::keys; using bptree_base::node; using bptree_base::num_chldrn; @@ -1329,11 +1435,15 @@ class bp_tree using base::free; using base::insrt_child; + using base::leaf; using base::leaf_level; + using base::parent; + using base::right; using base::root; public: static constexpr auto unique{ true }; // TODO non unique + static constexpr auto transparent_comparator{ requires{ typename Comparator::is_transparent; } }; using size_type = base::size_type; using value_type = base::value_type; @@ -1389,7 +1499,7 @@ class bp_tree if ( locations.leaf_offset.exact_find ) [[ unlikely ]] return { { this->nodes_, { slot_of( locations.leaf ), locations.leaf_offset.pos } }, false }; - auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, v, { /*insertion starts from leaves which do not have children*/ } ) }; + auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; ++this->hdr().size_; return { std::prev( iterator{ this->nodes_, { insert_pos_next.node, insert_pos_next.next_insert_offset } } ), true }; } @@ -1405,19 +1515,11 @@ class bp_tree size_type merge( bp_tree & other ); - [[ using gnu: noinline, pure, sysv_abi ]] - const_iterator find( key_const_arg key ) const noexcept - { - if ( !empty() ) [[ likely ]] - { - auto const location{ const_cast( *this ).find_nodes_for( key ) }; - if ( location.leaf_offset.exact_find ) [[ likely ]] { - return iterator{ const_cast( *this ).nodes_, { slot_of( location.leaf ), location.leaf_offset.pos } }; - } - } - - return this->cend(); - } + [[ nodiscard ]] bool contains ( KeyType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key }; } + [[ nodiscard ]] iterator find ( KeyType auto const & key ) noexcept { return find_impl ( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iterator find ( KeyType auto const & key ) const noexcept { return const_cast( *this ).find( key ); } + [[ nodiscard ]] iterator lower_bound( KeyType auto const & key ) noexcept { return lower_bound_impl( pass_in_reg{ key }; } + [[ nodiscard ]] const_iterator lower_bound( KeyType auto const & key ) const noexcept { return const_cast( *this ).lower_bound( key ); } [[ nodiscard ]] BOOST_NOINLINE bool erase( key_const_arg key ) noexcept @@ -1483,19 +1585,48 @@ class bp_tree void swap( bp_tree & other ) noexcept { base::swap( other ); } - Comparator const & comp() const noexcept { return *this; } + [[ nodiscard ]] Comparator const & comp() const noexcept { return *this; } // UB if the comparator is changed in such a way as to invalidate to order of elements already in the container [[ nodiscard ]] Comparator & mutable_comp() noexcept { return *this; } +private: // pass-in-reg public function overloads/impls + bool contains_impl( Reg auto const key ) const noexcept { return !empty() && const_cast( *this ).find_nodes_for( key ).leaf_offset.exact_find; } + + [[ using gnu: noinline, pure, sysv_abi ]] + iterator find_impl( Reg auto const key ) noexcept + { + if ( !empty() ) [[ likely ]] + { + auto const location{ find_nodes_for( key ) }; + if ( location.leaf_offset.exact_find ) [[ likely ]] { + return iterator{ this->nodes_, { slot_of( location.leaf ), location.leaf_offset.pos } }; + } + } + + return this->end(); + } + + iterator lower_bound_impl( Reg auto const key ) noexcept + { + if ( !empty() ) [[ likely ]] + { + auto const location{ find_nodes_for( key ) }; + return iterator{ this->nodes_, { slot_of( location.leaf ), location.leaf_offset.pos } }; + } + + return this->end(); + } + private: + // lower_bound find struct find_pos // msvc pass-in-reg facepalm { node_size_type pos : ( sizeof( node_size_type ) * CHAR_BIT - 1 ); node_size_type exact_find : 1; }; [[ using gnu: pure, hot, noinline, sysv_abi ]] - find_pos find( Key const keys[], node_size_type const num_vals, key_const_arg value ) const noexcept + find_pos find( Key const keys[], node_size_type const num_vals, Reg auto const value ) const noexcept { // TODO branchless binary search, Alexandrescu's ideas, https://orlp.net/blog/bitwise-binary-search ... BOOST_ASSUME( num_vals > 0 ); @@ -1516,16 +1647,14 @@ class bp_tree auto const exact_find{ ( pos_idx != num_vals ) & !comp( value, keys[ std::min( pos_idx, num_vals - 1 ) ] ) }; return { pos_idx, reinterpret_cast( exact_find ) }; } - auto find( auto const & node, key_const_arg value ) const noexcept + find_pos find( auto const & node, auto const & value ) const noexcept { return find( node.keys, node.num_vals, pass_in_reg{ value } ); } + [[ using gnu: pure, hot, sysv_abi ]] + find_pos find_with_offset( auto const & node, node_size_type const offset, Reg auto const value ) const noexcept { - return find( node.keys, node.num_vals, value ); - } - - template - void insert( N & target_node, key_const_arg v, node_slot const right_child ) - { - auto const pos{ find( target_node, v ).pos }; - base::insert( target_node, pos, v, right_child ); + BOOST_ASSUME( offset < node.num_vals ); + auto result{ find( &node.keys[ offset ], node.num_vals - offset, value ) }; + result.pos += offset; + return result; } struct key_locations @@ -1538,13 +1667,13 @@ class bp_tree }; [[ using gnu: pure, hot, sysv_abi ]] - key_locations find_nodes_for( key_const_arg key ) noexcept + key_locations find_nodes_for( Reg auto const key ) noexcept { node_slot separator_key_node; node_size_type separator_key_offset{}; // a leaf (lone) root is implicitly handled by the loop condition: - // depth_ == 1 so the loop is skipped entirely and the lone root is never examined - // through the incorrectly typed reference + // depth_ == 1 so the loop is skipped entirely and the lone root is + // never examined through the incorrectly typed reference auto p_node{ &bptree_base::as( root() ) }; auto const depth { this->hdr().depth_ }; BOOST_ASSUME( depth >= 1 ); @@ -1554,7 +1683,7 @@ class bp_tree if ( exact_find ) { // separator key - it also means we have to traverse to the right - BOOST_ASSUME( !separator_key_node ); + BOOST_ASSUME( !separator_key_node ); // exact_find may happen at most once separator_key_node = slot_of( *p_node ); separator_key_offset = pos; ++pos; // traverse to the right child @@ -1570,6 +1699,77 @@ class bp_tree separator_key_node }; } + key_locations find_nodes_for( Key const & key ) noexcept { return find_nodes_for( pass_in_reg{ key } ); } + + auto find_next( leaf_node const & starting_leaf, node_size_type const starting_leaf_offset, Reg auto const key ) const noexcept + { + if ( leq( key, keys( starting_leaf ).back() ) ) + { + auto const pos{ find_with_offset( starting_leaf, starting_leaf_offset, key ) }; + BOOST_ASSUME( pos.pos != starting_leaf.num_vals ); + BOOST_ASSUME( pos.pos >= starting_leaf_offset ); + return std::make_pair( const_cast( &starting_leaf ), pos ); + } + + if ( !starting_leaf.right ) [[ unlikely ]] // we are at the end of the tree/leaf level: key not present at all + return std::make_pair( const_cast( &starting_leaf ), find_pos{ starting_leaf.num_vals, false } ); + + // key in tree but not in starting leaf: go up the tree + auto const * prnt{ &parent( starting_leaf ) }; + auto parent_offset{ starting_leaf.parent_child_idx }; + BOOST_ASSUME( ( parent_offset == prnt->num_vals ) || ge( prnt->keys[ parent_offset ], starting_leaf.keys[ 0 ] ) ); + auto const depth{ this->hdr().depth_ }; BOOST_ASSUME( depth >= 1 ); + auto level{ depth - 1 }; + while ( le( keys( *prnt ).back(), key ) ) + { + if ( level == 1 ) [[ unlikely ]] + { + // reached the root + BOOST_ASSUME( !prnt->parent ); + // the case where the key does not exist at all is handled at + // the beginning so the only case where parent_offset could + // point to the end is on intermediate inner/parent nodes (when + // depth is more then 2 levels) + // (and this has to be handled because otherwise the + // find_with_offset call below would get fed empty input which + // it does not support) + BOOST_ASSUME( depth > 2 || ( parent_offset < prnt->num_vals ) ); + parent_offset = std::min( parent_offset, node_size_type( prnt->num_vals - 1 ) ); + break; + } + parent_offset = prnt->parent_child_idx; + prnt = &parent( *prnt ); + --level; + } + BOOST_ASSUME( parent_offset < prnt->num_vals ); + // descend to the leaf containing the key + for ( ; level < depth; ++level ) + { + auto [pos, exact_find]{ find_with_offset( *prnt, parent_offset, key ) }; + BOOST_ASSERT( !exact_find ); + pos += exact_find; // traverse to the right child for separator keys + prnt = &node( children( *prnt )[ pos ] ); + parent_offset = 0; + } + BOOST_ASSUME( parent_offset == 0 ); + auto const & containing_leaf{ as( *prnt ) }; + auto const pos{ find( containing_leaf, key ) }; + BOOST_ASSUME + ( + ( &starting_leaf != &containing_leaf ) || + // the worst case: when the value falls between existing nodes we + // will land on the starting node again - TODO insert a new node + ( pos.pos == containing_leaf.num_vals ) + ); + return std::make_pair( const_cast( &containing_leaf ), pos ); + } + + template + void insert( N & target_node, key_const_arg v, node_slot const right_child ) + { + auto const pos{ find( target_node, v ).pos }; + base::insert( target_node, pos, v, right_child ); + } size_type insert( typename base::bulk_copied_input ); @@ -1613,6 +1813,7 @@ class bp_tree #endif [[ gnu::pure ]] bool le( key_const_arg left, key_const_arg right ) const noexcept { return comp()( left, right ); } + [[ gnu::pure ]] bool ge( key_const_arg left, key_const_arg right ) const noexcept { return comp()( right, left ); } [[ gnu::pure ]] bool eq( key_const_arg left, key_const_arg right ) const noexcept { if constexpr ( requires{ comp().eq( left, right ); } ) @@ -1625,10 +1826,14 @@ class bp_tree { if constexpr ( requires{ comp().leq( left, right ); } ) return comp().leq( left, right ); - if constexpr ( detail::is_simple_comparator && requires { left == right; } ) - return left <= right; return !comp()( right, left ); } + [[ gnu::pure ]] bool geq( key_const_arg left, key_const_arg right ) const noexcept + { + if constexpr ( requires{ comp().geq( left, right ); } ) + return comp().geq( left, right ); + return !comp()( left, right ); + } }; // class bp_tree PSI_WARNING_DISABLE_POP() @@ -1654,7 +1859,8 @@ auto bp_tree::merge auto const & new_separator{ src_keys[ 0 ] }; BOOST_ASSUME( new_separator < tgt_keys[ 0 ] ); base::update_separator( target, new_separator ); - // TODO rather simply insert the source leaf into the parent + // TODO rather simply insert the source leaf into the parent (if all of + // its keys come before the first key in target) } if ( !available_space ) [[ unlikely ]] { @@ -1665,23 +1871,18 @@ auto bp_tree::merge { return std::make_tuple( 0, 1, &target, target_offset ); } - auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, src_keys[ 0 ], {} ) }; - auto const & src{ source_slot ? base::leaf( source_slot ) : source }; - auto & tgt{ base::leaf( target_slot ) }; + auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, pass_rv_in_reg{ auto{ src_keys[ 0 ] } }, {} ) }; + auto const & src{ source_slot ? leaf( source_slot ) : source }; + auto & tgt{ leaf( target_slot ) }; BOOST_ASSUME( next_tgt_offset <= tgt.num_vals ); auto const next_src_offset{ static_cast( source_offset + 1 ) }; // next_tgt_offset returned by split_to_insert points to the // position in the target node that immediately follows the // position for the inserted src_keys[ 0 ] - IOW it need not be // the position for src.keys[ next_src_offset ] - if ( tgt.num_vals != next_tgt_offset ) // necessary check because find assumes non-empty input + if ( next_tgt_offset != tgt.num_vals ) // necessary check because find assumes non-empty input { - next_tgt_offset += find - ( - &tgt.keys[ next_tgt_offset ], - tgt.num_vals - next_tgt_offset, - src.keys[ next_src_offset ] - ).pos; + next_tgt_offset = find_with_offset( tgt, next_tgt_offset, pass_in_reg{ src.keys[ next_src_offset ] } ).pos; } return std::make_tuple( 1, 1, &tgt, next_tgt_offset ); } @@ -1692,12 +1893,13 @@ auto bp_tree::merge // size accordingly to maintain the sorted property). if ( target.right ) { - auto const & right_delimiter { base::leaf( target.right ).keys[ 0 ] }; - auto const less_than_right_pos{ find( src_keys, copy_size, right_delimiter ).pos }; - if ( less_than_right_pos != copy_size ) + auto const & right_delimiter { right( target ).keys[ 0 ] }; + auto const less_than_right_pos{ find( src_keys, copy_size, pass_in_reg{ right_delimiter } ) }; + BOOST_ASSUME( !less_than_right_pos.exact_find ); + if ( less_than_right_pos.pos != copy_size ) { - BOOST_ASSUME( less_than_right_pos < copy_size ); - node_size_type const input_end_for_target( less_than_right_pos + source_offset ); + BOOST_ASSUME( less_than_right_pos.pos < copy_size ); + node_size_type const input_end_for_target( less_than_right_pos.pos + source_offset ); BOOST_ASSUME( input_end_for_target > source_offset ); BOOST_ASSUME( input_end_for_target <= source.num_vals ); copy_size = static_cast( input_end_for_target - source_offset ); @@ -1706,28 +1908,33 @@ auto bp_tree::merge auto & tgt_size{ target.num_vals }; node_size_type inserted_size; + node_size_type next_tgt_offset; if ( target_offset == tgt_size ) // a simple append { std::copy_n( src_keys, copy_size, &tgt_keys[ target_offset ] ); - tgt_size += copy_size; - inserted_size = copy_size; + tgt_size += copy_size; + inserted_size = copy_size; + next_tgt_offset = tgt_size; } else { BOOST_ASSUME( copy_size + tgt_size <= leaf_node::max_values ); - std::move_backward( &tgt_keys[ 0 ], &tgt_keys[ tgt_size ], &tgt_keys[ tgt_size + copy_size ] ); - auto const new_tgt_size{ merge_interleaved_values + // make room for merge: move existing values (beyond the insertion/merge + // point) to the end of the buffer + std::move_backward( &tgt_keys[ target_offset ], &tgt_keys[ tgt_size ], &tgt_keys[ tgt_size + copy_size ] ); + auto const new_tgt_size{ target_offset + merge_interleaved_values ( - &src_keys[ 0 ], copy_size, - &tgt_keys[ copy_size ], tgt_size, - &tgt_keys[ 0 ] + &src_keys[ 0 ], copy_size, + &tgt_keys[ target_offset + copy_size ], tgt_size - target_offset, + &tgt_keys[ target_offset ] ) }; - inserted_size = static_cast( new_tgt_size - tgt_size ); - tgt_size = new_tgt_size; + inserted_size = static_cast( new_tgt_size - tgt_size ); + tgt_size = new_tgt_size; + next_tgt_offset = target_offset + 1; } verify( target ); BOOST_ASSUME( inserted_size <= copy_size ); - return std::make_tuple( inserted_size, copy_size, &target, tgt_size ); + return std::make_tuple( inserted_size, copy_size, &target, next_tgt_offset ); } @@ -1750,24 +1957,27 @@ bp_tree::insert( typename base::bulk_copied_input const input ) return total_size; } + auto p_new_keys{ p_new_nodes_begin }; + auto [source_slot, source_slot_offset]{ p_new_keys.pos() }; + auto src_leaf{ &leaf( source_slot ) }; + + auto const start_pos{ find_nodes_for( *p_new_keys ) }; + auto tgt_leaf { &start_pos.leaf }; + auto tgt_leaf_next_pos{ start_pos.leaf_offset }; + size_type inserted{ 0 }; - for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) + do { - auto const tgt_location{ find_nodes_for( *p_new_keys ) }; - if ( tgt_location.leaf_offset.exact_find ) [[ unlikely ]] + if ( unique && tgt_leaf_next_pos.exact_find ) [[ unlikely ]] { ++p_new_keys; continue; } - auto tgt_leaf { &tgt_location.leaf }; - auto tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; - auto const [source_slot, source_slot_offset]{ p_new_keys.pos() }; - auto * src_leaf{ &this->leaf( source_slot ) }; BOOST_ASSUME( source_slot_offset < src_leaf->num_vals ); // if we have reached the end of the rightmost leaf simply perform // a bulk_append - if ( ( tgt_leaf_next_pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) + if ( ( tgt_leaf_next_pos.pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) { auto const so_far_consumed{ static_cast( p_new_keys - p_new_nodes_begin ) }; BOOST_ASSUME( so_far_consumed < total_size ); @@ -1781,35 +1991,47 @@ bp_tree::insert( typename base::bulk_copied_input const input ) inserted += static_cast( total_size - so_far_consumed ); break; } + // TODO in-the-middle partial bulk-inserts - node_size_type inserted_count, consumed_source; - std::tie( inserted_count, consumed_source, tgt_leaf, tgt_leaf_next_pos ) = + auto const [inserted_count, consumed_source, tgt_next_leaf, tgt_next_offset] + { merge ( *src_leaf, source_slot_offset, - *tgt_leaf, tgt_leaf_next_pos - ); - // TODO merge returns 'hints' for the next target position: use that - // for a faster version of find_nodes_for (that does not search from - // the beginning every time). + *tgt_leaf, tgt_leaf_next_pos.pos + ) + }; + tgt_leaf = tgt_next_leaf; // merge might have caused a relocation (by calling split_to_insert) // TODO use iter_pos directly p_new_keys .update_pool_ptr( this->nodes_ ); p_new_nodes_end.update_pool_ptr( this->nodes_ ); + src_leaf = &leaf( source_slot ); + p_new_keys += consumed_source; inserted += inserted_count; - if ( source_slot != p_new_keys.pos().node ) + + if ( source_slot != p_new_keys.pos().node ) // have we moved to the next node? { // merged leaves (their contents) were effectively copied into // existing leaves (instead of simply linked into the tree // structure) and now have to be returned to the free pool // TODO: add leak detection to/for the entire bp_tree class - src_leaf = &this->leaf( source_slot ); // could be invalidated by split_to_insert in merge base::unlink_right( *src_leaf ); free( *src_leaf ); + + source_slot = p_new_keys.pos().node; + src_leaf = &leaf( source_slot ); } - } + source_slot_offset = p_new_keys.pos().value_offset; + + // seek the next position starting from the current one (relying on the + // that we are using presorted data) rather than starting everytime from + // scratch (using find_nodes_for) + std::tie( tgt_leaf, tgt_leaf_next_pos ) = + find_next( *tgt_leaf, tgt_next_offset, pass_in_reg{ src_leaf->keys[ source_slot_offset ] } ); + } while ( p_new_keys != p_new_nodes_end ); BOOST_ASSUME( inserted <= total_size ); this->hdr().size_ += inserted; @@ -1823,7 +2045,7 @@ bp_tree::merge( bp_tree & other ) // This function follows nearly the same logic as bulk insert (consult it // for more comments), the main differences being: // - no need to copy and sort the input - // - the bulk_append phase has to first copy the remaining of the source + // - the bulk_append phase has to first copy the remainder of the source // nodes (they are not somehow 'extractable' from the source tree) // - care has to be taken around the fact that source leaves are coming // from a different tree instance (i.e. from a different container) e.g. diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index ffa9e49..975e7c9 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -40,7 +40,21 @@ bptree_base::user_header_data() noexcept { return header_data().second; } bptree_base::header & bptree_base::hdr() noexcept { return *header_data().first; } -void bptree_base::reserve( node_slot::value_type additional_nodes ) +bptree_base::storage_result +bptree_base::map_memory( std::uint32_t const initial_capacity_as_number_of_nodes ) noexcept +{ + storage_result success{ nodes_.map_memory( initial_capacity_as_number_of_nodes, value_init ) }; + if ( std::move( success ) ) + { + hdr() = {}; + if ( initial_capacity_as_number_of_nodes ) { + assign_nodes_to_free_pool( 0 ); + } + } + return success; +} + +void bptree_base::reserve_additional( node_slot::value_type additional_nodes ) { auto const preallocated_count{ hdr().free_node_count_ }; additional_nodes -= std::min( preallocated_count, additional_nodes ); @@ -49,9 +63,21 @@ void bptree_base::reserve( node_slot::value_type additional_nodes ) #ifndef NDEBUG hdr_ = &hdr(); #endif - assign_nodes_to_free_pool( static_cast( current_size ) ); + assign_nodes_to_free_pool( current_size ); } +void bptree_base::reserve( node_slot::value_type new_capacity_in_number_of_nodes ) +{ + if ( new_capacity_in_number_of_nodes <= nodes_.capacity() ) + return; + auto const current_size{ nodes_.size() }; + nodes_.grow_to( new_capacity_in_number_of_nodes, value_init ); +#ifndef NDEBUG + hdr_ = &hdr(); +#endif + assign_nodes_to_free_pool( current_size ); +} +[[ gnu::cold ]] void bptree_base::assign_nodes_to_free_pool( node_slot::value_type const starting_node ) noexcept { for ( auto & n : std::views::reverse( std::span{ nodes_ }.subspan( starting_node ) ) ) From de81cf205c09649a1d6e8776e302f4d245f940f3 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Tue, 22 Oct 2024 10:44:34 +0200 Subject: [PATCH 39/49] Fixed map_memory to correctly handle data initialization. --- include/psi/vm/vector.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index 2c7953f..e66200c 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -1084,7 +1084,16 @@ class vector /////////////////////////////////////////////////////////////////////////// auto map_file ( auto const file, flags::named_object_construction_policy const policy ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_file ( file, policy ); } - auto map_memory( size_type const initial_size = 0 ) noexcept { BOOST_ASSERT( !has_attached_storage() ); return storage_.map_memory( to_byte_sz( initial_size ) ); } + template + auto map_memory( size_type const initial_size = 0, InitPolicy init_policy = {} ) noexcept + { + BOOST_ASSERT( !has_attached_storage() ); + auto result{ storage_.map_memory( to_byte_sz( initial_size ) ) }; + if ( std::is_same_v && initial_size && std::move( result ) ) { + std::uninitialized_default_construct( begin(), end() ); + } + return result; + } bool has_attached_storage() const noexcept { return static_cast( storage_ ); } @@ -1171,7 +1180,12 @@ class vector PSI_WARNING_GCC_OR_CLANG_DISABLE( -Wsign-conversion ) static T * to_t_ptr ( mapped_view::value_type * const ptr ) noexcept { return reinterpret_cast( ptr ); } static sz_t to_t_sz ( auto const byte_sz ) noexcept { BOOST_ASSUME( byte_sz % sizeof( T ) == 0 ); return static_cast( byte_sz / sizeof( T ) ); } - static sz_t to_byte_sz( auto const sz ) noexcept { return static_cast( sz * sizeof( T ) ); } + static sz_t to_byte_sz( auto const sz ) noexcept + { + auto const rez{ sz * sizeof( T ) }; + BOOST_ASSERT( rez <= std::numeric_limits::max() ); + return static_cast( rez ); + } PSI_WARNING_DISABLE_POP() void shrink_storage_to( size_type const target_size ) noexcept From 1719f8f168f3674778846e97d729d2dea32ed48d Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Tue, 22 Oct 2024 12:28:40 +0200 Subject: [PATCH 40/49] Updated merge() to use find_next() like insert() already does. Added hinted insert(). Minor related cleanups and refactoring. --- include/psi/vm/containers/b+tree.hpp | 116 +++++++++++++++++---------- 1 file changed, 72 insertions(+), 44 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 742e06b..2131156 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -115,7 +115,7 @@ struct [[ clang::trivial_abi ]] pass_rv_in_reg }; // pass_rv_in_reg template -concept KeyType = transparent_comparator || std::is_same_v; +concept KeyType = ( transparent_comparator && std::is_convertible_v ) || std::is_same_v; template bool constexpr reg { false }; template bool constexpr reg>{ true }; @@ -1482,27 +1482,15 @@ class bp_tree auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } - BOOST_NOINLINE - std::pair insert( key_const_arg v ) - { - if ( empty() ) - { - auto & root{ static_cast( base::create_root() ) }; - BOOST_ASSUME( root.num_vals == 1 ); - root.keys[ 0 ] = v; - return { begin(), true }; - } + [[ nodiscard ]] bool contains ( KeyType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key }; } + [[ nodiscard ]] iterator find ( KeyType auto const & key ) noexcept { return find_impl ( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iterator find ( KeyType auto const & key ) const noexcept { return const_cast( *this ).find( key ); } + [[ nodiscard ]] iterator lower_bound( KeyType auto const & key ) noexcept { return lower_bound_impl( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iterator lower_bound( KeyType auto const & key ) const noexcept { return const_cast( *this ).lower_bound( key ); } - auto const locations{ find_nodes_for( v ) }; - BOOST_ASSUME( !locations.inner ); - BOOST_ASSUME( !locations.inner_offset ); - if ( locations.leaf_offset.exact_find ) [[ unlikely ]] - return { { this->nodes_, { slot_of( locations.leaf ), locations.leaf_offset.pos } }, false }; + std::pair insert( KeyType auto const & key ) { return insert_impl( pass_in_reg{ key } ); } - auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; - ++this->hdr().size_; - return { std::prev( iterator{ this->nodes_, { insert_pos_next.node, insert_pos_next.next_insert_offset } } ), true }; - } + iterator insert( const_iterator const pos_hint, KeyType auto const & key ) { return insert_impl( pos_hint, pass_in_reg{ key } ); } // bulk insert // performance note: insertion of existing values into a unique bp_tree is @@ -1513,13 +1501,7 @@ class bp_tree size_type insert( InIter const begin, InIter const end ) { return insert( base::bulk_insert_prepare( std::ranges::subrange( begin, end ) ) ); } size_type insert( std::ranges::range auto const & keys ) { return insert( base::bulk_insert_prepare( std::ranges::subrange( keys ) ) ); } - size_type merge( bp_tree & other ); - - [[ nodiscard ]] bool contains ( KeyType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key }; } - [[ nodiscard ]] iterator find ( KeyType auto const & key ) noexcept { return find_impl ( pass_in_reg{ key } ); } - [[ nodiscard ]] const_iterator find ( KeyType auto const & key ) const noexcept { return const_cast( *this ).find( key ); } - [[ nodiscard ]] iterator lower_bound( KeyType auto const & key ) noexcept { return lower_bound_impl( pass_in_reg{ key }; } - [[ nodiscard ]] const_iterator lower_bound( KeyType auto const & key ) const noexcept { return const_cast( *this ).lower_bound( key ); } + size_type merge( bp_tree && other ); [[ nodiscard ]] BOOST_NOINLINE bool erase( key_const_arg key ) noexcept @@ -1618,6 +1600,40 @@ class bp_tree return this->end(); } + std::pair insert_impl( Reg auto const v ) + { + if ( empty() ) + { + auto & root{ static_cast( base::create_root() ) }; + BOOST_ASSUME( root.num_vals == 1 ); + root.keys[ 0 ] = v; + return { begin(), true }; + } + + auto const locations{ find_nodes_for( v ) }; + BOOST_ASSUME( !locations.inner ); + BOOST_ASSUME( !locations.inner_offset ); + if ( locations.leaf_offset.exact_find ) [[ unlikely ]] + return { { this->nodes_, { slot_of( locations.leaf ), locations.leaf_offset.pos } }, false }; + + auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; + ++this->hdr().size_; + return { std::prev( iterator{ this->nodes_, { insert_pos_next.node, insert_pos_next.next_insert_offset } } ), true }; + } + + iterator insert_impl( const_iterator const pos_hint, Reg auto const v ) + { + // yes, for starters generic 'hint as just a hint' is not supported + BOOST_ASSUME( !empty() ); + BOOST_ASSERT_MSG( le( v, *pos_hint ), "Invalid insertion hint" ); + BOOST_ASSERT_MSG( !unique || ge( v, *std::prev( pos_hint ) ), "Invalid insertion hint" ); + + auto const [hint_slot, hint_slot_offset]{ pos_hint.base().pos() }; + auto const insert_pos_next{ base::insert( leaf( hint_slot ), hint_slot_offset, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; + ++this->hdr().size_; + return std::prev( iterator{ this->nodes_, { insert_pos_next.node, insert_pos_next.next_insert_offset } } ); + } + private: // lower_bound find struct find_pos // msvc pass-in-reg facepalm @@ -2040,7 +2056,7 @@ bp_tree::insert( typename base::bulk_copied_input const input ) template bp_tree::size_type -bp_tree::merge( bp_tree & other ) +bp_tree::merge( bp_tree && other ) { // This function follows nearly the same logic as bulk insert (consult it // for more comments), the main differences being: @@ -2057,28 +2073,31 @@ bp_tree::merge( bp_tree & other ) } auto const total_size{ other.size() }; - this->reserve( total_size ); + this->reserve_additional( total_size ); auto const p_new_nodes_begin{ other.ra_begin() }; auto const p_new_nodes_end { other.ra_end () }; + auto p_new_keys{ p_new_nodes_begin }; + auto src_leaf { &other.leaf( p_new_keys.pos().node ) }; + auto source_slot_offset{ p_new_keys.pos().value_offset }; + + auto const start_pos{ find_nodes_for( *p_new_keys ) }; + auto tgt_leaf { &start_pos.leaf }; + auto tgt_leaf_next_pos{ start_pos.leaf_offset }; + size_type inserted{ 0 }; - for ( auto p_new_keys{ p_new_nodes_begin }; p_new_keys != p_new_nodes_end; ) + do { - auto const tgt_location{ find_nodes_for( *p_new_keys ) }; - if ( tgt_location.leaf_offset.exact_find ) [[ unlikely ]] + if ( unique && tgt_leaf_next_pos.exact_find ) [[ unlikely ]] { ++p_new_keys; continue; } - auto & tgt_leaf { tgt_location.leaf }; - auto const tgt_leaf_next_pos{ tgt_location.leaf_offset.pos }; - auto * src_leaf { &other.leaf( p_new_keys.pos().node ) }; - auto const source_slot_offset{ p_new_keys.pos().value_offset }; BOOST_ASSUME( source_slot_offset < src_leaf->num_vals ); // simple bulk_append at the end of the rightmost leaf - if ( ( tgt_leaf_next_pos == tgt_leaf.num_vals ) && !tgt_leaf.right ) + if ( ( tgt_leaf_next_pos.pos == tgt_leaf->num_vals ) && !tgt_leaf->right ) { // pre-copy the (remainder of the) source into fresh nodes in // order to simply call bulk_append @@ -2093,7 +2112,7 @@ bp_tree::merge( bp_tree & other ) this->move_keys( *src_leaf, source_slot_offset, src_leaf->num_vals, src_leaf_copy, 0 ); src_leaf_copy.num_vals = src_leaf->num_vals - source_slot_offset; src_leaf->num_vals = source_slot_offset; - this->link( tgt_leaf, src_leaf_copy ); + this->link( *tgt_leaf, src_leaf_copy ); this->bulk_append_fill_incomplete_leaf( src_leaf_copy ); } else @@ -2114,22 +2133,31 @@ bp_tree::merge( bp_tree & other ) auto const so_far_consumed{ static_cast( p_new_keys - p_new_nodes_begin ) }; BOOST_ASSUME( so_far_consumed < total_size ); - this->bulk_append( &this->leaf( src_copy_begin ), { tgt_leaf.parent, tgt_leaf.parent_child_idx } ); + this->bulk_append( &this->leaf( src_copy_begin ), { tgt_leaf->parent, tgt_leaf->parent_child_idx } ); inserted += static_cast( total_size - so_far_consumed ); break; } + // TODO in-the-middle partial bulk-inserts - node_size_type inserted_count, consumed_source; - std::tie( inserted_count, consumed_source, std::ignore, std::ignore ) = + auto const [inserted_count, consumed_source, tgt_next_leaf, tgt_next_offset] + { merge ( *src_leaf, source_slot_offset, - tgt_leaf, tgt_leaf_next_pos - ); + *tgt_leaf, tgt_leaf_next_pos.pos + ) + }; + tgt_leaf = tgt_next_leaf; p_new_keys += consumed_source; inserted += inserted_count; - } // main loop + + src_leaf = &other.leaf( p_new_keys.pos().node ); + source_slot_offset = p_new_keys.pos().value_offset; + + std::tie( tgt_leaf, tgt_leaf_next_pos ) = + find_next( *tgt_leaf, tgt_next_offset, pass_in_reg{ src_leaf->keys[ source_slot_offset ] } ); + } while ( p_new_keys != p_new_nodes_end ); BOOST_ASSUME( inserted <= total_size ); this->hdr().size_ += inserted; From 13bc5d1fb7a7cd56de54a59dd7906a6012029fb5 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Tue, 22 Oct 2024 12:44:32 +0200 Subject: [PATCH 41/49] Updated bp_tree tests. --- test/b+tree.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/b+tree.cpp b/test/b+tree.cpp index f3d2189..41e61b5 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -35,8 +35,7 @@ TEST( bp_tree, playground ) std::ranges::shuffle( nums.subspan( 3 * nums.size() / 4 ), rng ); { bp_tree bpt; - bpt.map_memory(); - bpt.reserve( nums.size() ); + bpt.map_memory( nums.size() ); { auto const third{ nums.size() / 3 }; auto const first_third{ nums.subspan( 0 * third, third ) }; @@ -66,6 +65,11 @@ TEST( bp_tree, playground ) EXPECT_EQ ( *bpt.insert( 42 ).first, 42 ); EXPECT_TRUE( std::ranges::equal( bpt.random_access() , sorted_numbers ) ); EXPECT_TRUE( std::ranges::equal( std::views::reverse( bpt.random_access() ), std::views::reverse( sorted_numbers ) ) ); + + EXPECT_TRUE( bpt.erase( 42 ) ); + auto const hint42{ bpt.lower_bound( 42 ) }; + EXPECT_EQ( *hint42, 42 + 1 ); + EXPECT_EQ( *bpt.insert( hint42, 42 ), 42 ); { auto const ra{ bpt.random_access() }; for ( auto n : std::views::iota( 0, test_size / 55 ) ) // slow operation (not really amortized constant time): use a smaller subset of the input @@ -93,7 +97,7 @@ TEST( bp_tree, playground ) shuffled_even_numbers.append_range( merge_appendix ); bpt_even.insert( shuffled_even_numbers ); - EXPECT_EQ( bpt.merge( bpt_even ), bpt_even.size() ); + EXPECT_EQ( bpt.merge( std::move( bpt_even ) ), bpt_even.size() ); } EXPECT_TRUE( std::ranges::equal( std::ranges::iota_view{ 0, test_size + extra_entries_for_tree_merge }, bpt ) ); From d5c38917be337efde4ee06d4aa92c098b4369d47 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Tue, 22 Oct 2024 13:02:03 +0200 Subject: [PATCH 42/49] MSVC: quick-fixed compilation errors. --- include/psi/vm/containers/b+tree.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 2131156..27590bc 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -864,7 +864,7 @@ class bptree_base_wkey : public bptree_base ( rightmost_parent, rightmost_parent_pos.next_insert_offset, - key_rv_arg{ auto{ src_leaf->keys[ 0 ] } }, + key_rv_arg{ /*mrmlj*/Key{ src_leaf->keys[ 0 ] } }, slot_of( *src_leaf ) ); if ( !next_src_slot ) @@ -964,7 +964,7 @@ class bptree_base_wkey : public bptree_base first_root_right.parent_child_idx = 1; hdr->depth_ = 1; auto const first_unconnected_node{ first_root_right.right }; - new_root( begin_leaf, first_root_left.right, key_rv_arg{ auto{ first_root_right.keys[ 0 ] } } ); // may invalidate references + new_root( begin_leaf, first_root_left.right, key_rv_arg{ /*mrmlj*/Key{ first_root_right.keys[ 0 ] } } ); // may invalidate references hdr = &this->hdr(); BOOST_ASSUME( hdr->depth_ == 2 ); bulk_append( &leaf( first_unconnected_node ), { hdr->root_, 1 } ); @@ -1482,7 +1482,7 @@ class bp_tree auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } - [[ nodiscard ]] bool contains ( KeyType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key }; } + [[ nodiscard ]] bool contains ( KeyType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key } ); } [[ nodiscard ]] iterator find ( KeyType auto const & key ) noexcept { return find_impl ( pass_in_reg{ key } ); } [[ nodiscard ]] const_iterator find ( KeyType auto const & key ) const noexcept { return const_cast( *this ).find( key ); } [[ nodiscard ]] iterator lower_bound( KeyType auto const & key ) noexcept { return lower_bound_impl( pass_in_reg{ key } ); } @@ -1887,7 +1887,7 @@ auto bp_tree::merge { return std::make_tuple( 0, 1, &target, target_offset ); } - auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, pass_rv_in_reg{ auto{ src_keys[ 0 ] } }, {} ) }; + auto [target_slot, next_tgt_offset]{ base::split_to_insert( target, target_offset, pass_rv_in_reg{ /*mrmlj*/Key{ src_keys[ 0 ] } }, {} ) }; auto const & src{ source_slot ? leaf( source_slot ) : source }; auto & tgt{ leaf( target_slot ) }; BOOST_ASSUME( next_tgt_offset <= tgt.num_vals ); From 488cb077cbbc2b52c359d56e08a8d027402e38b6 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Tue, 22 Oct 2024 13:44:34 +0200 Subject: [PATCH 43/49] Clang: fixed compiler warnings (for the basic_const_iterator error regenerate CMake cache/refetch std_fix). --- include/psi/vm/containers/b+tree.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 27590bc..958d401 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -910,7 +910,7 @@ class bptree_base_wkey : public bptree_base leaf_node & leaf{ this->leaf( leaf_slot ) }; BOOST_ASSUME( leaf.num_vals == 0 ); if constexpr ( can_preallocate ) { - auto const size_to_copy{ static_cast( std::min( leaf.max_values, keys.end() - p_keys ) ) }; + auto const size_to_copy{ static_cast( std::min( leaf.max_values, static_cast( keys.end() - p_keys ) ) ) }; BOOST_ASSUME( size_to_copy ); std::copy_n( p_keys, size_to_copy, leaf.keys ); leaf.num_vals = size_to_copy; @@ -1945,7 +1945,7 @@ auto bp_tree::merge &tgt_keys[ target_offset ] ) }; inserted_size = static_cast( new_tgt_size - tgt_size ); - tgt_size = new_tgt_size; + tgt_size = static_cast( new_tgt_size ); next_tgt_offset = target_offset + 1; } verify( target ); From e0c45d2a9627f4d4135b1cf22ec801d7bfaa3e1e Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Wed, 23 Oct 2024 17:17:43 +0200 Subject: [PATCH 44/49] Added the public bp_tree::erase( iter ) method. Added the get_contiguous_span_and_move_to_next_node() 'extended' member function to iterators to ease implementing efficient algorithms w/o further breaking encapsulation. --- include/psi/vm/containers/b+tree.hpp | 241 ++++++++++++++++++--------- src/containers/b+tree.cpp | 10 +- test/b+tree.cpp | 4 + 3 files changed, 170 insertions(+), 85 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 958d401..f844967 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -262,6 +262,8 @@ class bptree_base protected: void swap( bptree_base & other ) noexcept; + base_iterator make_iter( iter_pos ) noexcept; + [[ gnu::pure ]] iter_pos begin_pos() const noexcept; [[ gnu::pure ]] iter_pos end_pos() const noexcept; @@ -430,7 +432,24 @@ inline constexpr bptree_base::node_slot const bptree_base::node_slot::null{ stat class bptree_base::base_iterator { +public: + constexpr base_iterator() noexcept = default; + + base_iterator & operator++() noexcept; + base_iterator & operator--() noexcept; + + bool operator==( base_iterator const & ) const noexcept; + +public: // extensions + iter_pos const & pos() const noexcept { return pos_; } + protected: + friend class bptree_base; + + base_iterator( node_pool &, iter_pos ) noexcept; + + [[ gnu::pure ]] node_header & node() const noexcept; + mutable #ifndef NDEBUG // for bounds checking std::span @@ -440,25 +459,10 @@ class bptree_base::base_iterator nodes_{}; iter_pos pos_ {}; - friend class bptree_base; - template friend class bp_tree; - - base_iterator( node_pool &, iter_pos ) noexcept; - - [[ gnu::pure ]] node_header & node() const noexcept; - +private: + template + friend class bp_tree; void update_pool_ptr( node_pool & ) const noexcept; - -public: - constexpr base_iterator() noexcept = default; - - base_iterator & operator++() noexcept; - base_iterator & operator--() noexcept; - - bool operator==( base_iterator const & ) const noexcept; - -public: - iter_pos const & pos() const noexcept { return pos_; } }; // class base_iterator //////////////////////////////////////////////////////////////////////////////// @@ -503,9 +507,20 @@ class bptree_base::base_random_access_iterator : public base_iterator template class bptree_base_wkey : public bptree_base { +private: + template + using iter_impl = boost::stl_interfaces::iterator_interface + < +# if !BOOST_STL_INTERFACES_USE_DEDUCED_THIS + Impl, +# endif + Tag, + Key + >; + public: - using key_type = Key; - using value_type = key_type; // TODO map support + using key_type = Key; + using value_type = key_type; // TODO map support // support for non trivial types (for which move and pass-by-ref matters) is WiP, nowhere near complete static_assert( std::is_trivial_v ); @@ -513,6 +528,12 @@ class bptree_base_wkey : public bptree_base using key_rv_arg = std::conditional_t, Key const, pass_rv_in_reg>; using key_const_arg = std::conditional_t, Key const, pass_in_reg >; + class fwd_iterator; + class ra_iterator; + + using iterator = fwd_iterator; + using const_iterator = std::basic_const_iterator; + public: storage_result map_memory( size_type initial_capacity = 0 ) noexcept { return bptree_base::map_memory( node_count_required_for_values( initial_capacity ) ); } size_type capacity() const noexcept @@ -541,7 +562,9 @@ class bptree_base_wkey : public bptree_base } void reserve_additional( size_type const additional_values ) { bptree_base::reserve_additional( node_count_required_for_values( additional_values ) ); } - void reserve ( size_type const new_capacity ) { bptree_base::reserve ( node_count_required_for_values( new_capacity ) ); }; + void reserve ( size_type const new_capacity ) { bptree_base::reserve ( node_count_required_for_values( new_capacity ) ); } + + iterator erase( const_iterator iter ) noexcept; // solely a debugging helper (include b+tree_print.hpp) void print() const; @@ -600,20 +623,6 @@ class bptree_base_wkey : public bptree_base static_assert( sizeof( inner_node ) == node_size ); static_assert( sizeof( leaf_node ) == node_size ); -protected: // iterators - template - using iter_impl = boost::stl_interfaces::iterator_interface - < -# if !BOOST_STL_INTERFACES_USE_DEDUCED_THIS - Impl, -# endif - Tag, - Key - >; - - class fwd_iterator; - class ra_iterator; - protected: // split_to_insert and its helpers root_node & new_root( node_slot const left_child, node_slot const right_child, key_rv_arg separator_key ) { @@ -831,6 +840,60 @@ class bptree_base_wkey : public bptree_base } } + iter_pos erase( leaf_node & leaf, node_size_type const leaf_key_offset ) noexcept + { + auto & hdr { this->hdr() }; + auto & depth_{ hdr.depth_ }; + + iter_pos next_pos{ slot_of( leaf ), leaf_key_offset }; + + lshift_keys( leaf, leaf_key_offset ); + --leaf.num_vals; + + if ( depth_ == 1 ) [[ unlikely ]] // handle 'leaf root' deletion directly to simplify handle_underflow() + { + auto & root_{ hdr.root_ }; + BOOST_ASSUME( root_ == slot_of( leaf ) ); + BOOST_ASSUME( leaf.is_root() ); + BOOST_ASSUME( hdr.size_ == leaf.num_vals + 1 ); + BOOST_ASSUME( !leaf.left ); + BOOST_ASSUME( !leaf.right ); + if ( leaf.num_vals == 0 ) + { + root_ = {}; + free( leaf ); + --depth_; + next_pos = bptree_base::end_pos(); + } + } + else + { + auto p_leaf{ &leaf }; + if ( underflowed( leaf ) ) + { + BOOST_ASSUME( !leaf.is_root() ); + BOOST_ASSUME( depth_ > 1 ); + next_pos = handle_underflow( leaf, leaf_level() ); + next_pos.value_offset += leaf_key_offset; + p_leaf = &this->leaf( next_pos.node ); + BOOST_ASSUME( next_pos.value_offset <= p_leaf->num_vals ); + } + + if ( leaf_key_offset == p_leaf->num_vals ) // the last value in a node was erased + { + if ( !p_leaf->right ) { + next_pos = bptree_base::end_pos(); + } else { + next_pos.node = p_leaf->right; + next_pos.value_offset = 0; + } + } + } + + --hdr.size_; + return next_pos; + } + // This function only serves the purpose of maintaining the rule about the // minimum number of children per node - that rule is actually only // 'academic' (for making sure that the performance/complexity guarantees @@ -972,9 +1035,9 @@ class bptree_base_wkey : public bptree_base hdr->size_ = total_size; } - leaf_node & leaf ( node_slot const slot ) noexcept { return node< leaf_node>( slot ); } - inner_node & inner ( node_slot const slot ) noexcept { return node( slot ); } - inner_node & parent( node_header & child ) noexcept { return inner( child.parent ); } + [[ gnu::pure ]] leaf_node & leaf ( node_slot const slot ) noexcept { return node< leaf_node>( slot ); } + [[ gnu::pure ]] inner_node & inner ( node_slot const slot ) noexcept { return node( slot ); } + [[ gnu::pure ]] inner_node & parent( node_header & child ) noexcept { return inner( child.parent ); } leaf_node const & leaf ( node_slot const slot ) const noexcept { return const_cast( *this ).leaf( slot ); } inner_node const & parent( node_header const & child ) const noexcept { return const_cast( *this ).parent( const_cast( child ) ); } @@ -1011,7 +1074,7 @@ class bptree_base_wkey : public bptree_base template BOOST_NOINLINE - void handle_underflow( N & node, depth_t const level ) noexcept + iter_pos handle_underflow( N & node, depth_t const level ) noexcept { BOOST_ASSUME( underflowed( node ) ); @@ -1043,6 +1106,11 @@ class bptree_base_wkey : public bptree_base auto const p_right_separator_key { has_right_sibling ? &parent.keys[ right_separator_key_idx ] : nullptr }; auto const p_left_separator_key { has_left_sibling ? &parent.keys[ left_separator_key_idx ] : nullptr }; + // save&return the node (and offset) that the underflowed node's values + // end up + auto final_node { node_slot }; + node_size_type final_node_original_keys_offset{ 0 }; + BOOST_ASSUME( has_right_sibling || has_left_sibling ); BOOST_ASSERT( &node != p_left_sibling ); BOOST_ASSERT( &node != p_right_sibling ); @@ -1079,6 +1147,8 @@ class bptree_base_wkey : public bptree_base p_left_sibling->num_vals--; verify( *p_left_sibling ); + final_node_original_keys_offset = 1; + BOOST_ASSUME( node. num_vals == N::min_values ); BOOST_ASSUME( p_left_sibling->num_vals >= N::min_values ); } @@ -1100,6 +1170,7 @@ class bptree_base_wkey : public bptree_base *p_right_separator_key = leftmost_right_key; } else { // Move/rotate the smallest key from the right sibling to the current node 'through' the parent + // no comparator in base classes :/ //BOOST_ASSUME( le( parent.keys[ parent_child_idx ], p_right_sibling->keys[ 0 ] ) ); //BOOST_ASSERT( le( *( node_keys.end() - 2 ), *p_right_separator_key ) ); @@ -1123,6 +1194,8 @@ class bptree_base_wkey : public bptree_base verify( *p_left_sibling ); BOOST_ASSUME( parent_has_key_copy == leaf_node_type ); // Merge node -> left sibling + final_node = node.left; + final_node_original_keys_offset = p_left_sibling->num_vals; merge_right_into_left( *p_left_sibling, node, parent, left_separator_key_idx, parent_child_idx ); } else { verify( *p_right_sibling ); @@ -1134,8 +1207,8 @@ class bptree_base_wkey : public bptree_base } // propagate underflow - auto & __restrict depth_{ this->hdr().depth_ }; - auto & __restrict root_ { this->hdr().root_ }; + auto & depth_{ this->hdr().depth_ }; + auto & root_ { this->hdr().root_ }; if ( parent.is_root() ) [[ unlikely ]] { BOOST_ASSUME( root_ == slot_of( parent ) ); @@ -1159,6 +1232,8 @@ class bptree_base_wkey : public bptree_base handle_underflow( parent, level - 1 ); } } + + return { final_node, final_node_original_keys_offset }; } // handle_underflow() root_node & root() noexcept { return as( bptree_base::root() ); } @@ -1308,6 +1383,19 @@ class bptree_base_wkey::fwd_iterator return leaf.keys[ pos_.value_offset ]; } + std::span get_contiguous_span_and_move_to_next_node() noexcept + { + auto & leaf{ static_cast( node() ) }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + std::span const span{ &leaf.keys[ pos_.value_offset ], leaf.num_vals - pos_.value_offset }; + if ( leaf.right ) [[ likely ]] + { + pos_.node = leaf.right; + pos_.value_offset = 0; + } + return span; + } + constexpr fwd_iterator & operator++() noexcept { return static_cast( base_iterator::operator++() ); } constexpr fwd_iterator & operator--() noexcept { return static_cast( base_iterator::operator--() ); } using impl::operator++; @@ -1332,6 +1420,7 @@ private: friend class bptree_base_wkey; public: constexpr ra_iterator() noexcept = default; + // TODO deduplicate this w/ fwd_iterator Key & operator*() const noexcept { auto & leaf{ node() }; @@ -1339,6 +1428,17 @@ private: friend class bptree_base_wkey; return leaf.keys[ pos_.value_offset ]; } + std::span get_contiguous_span_and_move_to_next_node() noexcept + { + auto & leaf{ static_cast( node() ) }; + BOOST_ASSUME( pos_.value_offset < leaf.num_vals ); + std::span const span{ &leaf.keys[ pos_.value_offset ], leaf.num_vals - pos_.value_offset }; + index_ += span.size(); + pos_.node = leaf.right; + pos_.value_offset = 0; + return span; + } + ra_iterator & operator+=( difference_type const n ) noexcept { return static_cast( base_random_access_iterator::operator+=( n ) ); } ra_iterator & operator++( ) noexcept { return static_cast( base_random_access_iterator::operator++( ) ); } @@ -1351,6 +1451,15 @@ private: friend class bptree_base_wkey; operator fwd_iterator() const noexcept { return static_cast( static_cast( *this ) ); } }; // class ra_iterator +template +typename +bptree_base_wkey::iterator +bptree_base_wkey::erase( const_iterator const iter ) noexcept +{ + auto const [node, key_offset]{ iter.base().pos() }; + return static_cast( make_iter( erase( leaf( node ), key_offset ) ) ); +} + template template [[ gnu::sysv_abi ]] void bptree_base_wkey::move_keys @@ -1451,8 +1560,8 @@ class bp_tree using const_pointer = value_type const *; using reference = value_type &; using const_reference = value_type const &; - using iterator = fwd_iterator; - using const_iterator = std::basic_const_iterator; + using iterator = base:: iterator; + using const_iterator = base::const_iterator; using base::empty; using base::size; @@ -1503,18 +1612,17 @@ class bp_tree size_type merge( bp_tree && other ); + using base::erase; [[ nodiscard ]] BOOST_NOINLINE bool erase( key_const_arg key ) noexcept { - auto & hdr{ base::hdr() }; - auto & __restrict depth_{ hdr.depth_ }; - auto & __restrict root_ { hdr.root_ }; - auto & __restrict size_ { hdr.size_ }; - auto const location{ find_nodes_for( key ) }; - // code below can go into base + + if ( !location.leaf_offset.exact_find ) [[ unlikely ]] + return false; + leaf_node & leaf{ location.leaf }; - if ( depth_ != 1 ) + if ( this->hdr().depth_ != 1 ) { verify( leaf ); BOOST_ASSUME( leaf.num_vals >= leaf.min_values ); @@ -1532,36 +1640,7 @@ class bp_tree separator_key = leaf.keys[ leaf_key_offset + 1 ]; } - - if ( !location.leaf_offset.exact_find ) [[ unlikely ]] - return false; - - lshift_keys( leaf, leaf_key_offset ); - BOOST_ASSUME( leaf.num_vals ); - --leaf.num_vals; - if ( depth_ == 1 ) [[ unlikely ]] // handle 'leaf root' deletion directly to simplify handle_underflow() - { - BOOST_ASSUME( root_ == slot_of( leaf ) ); - BOOST_ASSUME( leaf.is_root() ); - BOOST_ASSUME( size_ == leaf.num_vals + 1 ); - BOOST_ASSUME( !leaf.left ); - BOOST_ASSUME( !leaf.right ); - if ( leaf.num_vals == 0 ) - { - root_ = {}; - base::free( leaf ); - --depth_; - } - } - else - if ( base::underflowed( leaf ) ) - { - BOOST_ASSUME( !leaf.is_root() ); - BOOST_ASSUME( depth_ > 1 ); - base::handle_underflow( leaf, base::leaf_level() ); - } - - --size_; + base::erase( leaf, leaf_key_offset ); return true; } @@ -1861,7 +1940,7 @@ auto bp_tree::merge ( leaf_node const & source, node_size_type const source_offset, leaf_node & target, node_size_type const target_offset -) noexcept +) /*?*/noexcept { verify( source ); verify( target ); diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index 975e7c9..ef86e12 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -267,8 +267,8 @@ bptree_base::base_random_access_iterator::operator+=( difference_type const n ) } else { un -= ( available_offset + 1 ); // Here we don't have to perform the same check as in the - // fwd_iterator increment as (end) iterator comparison is done - // solely through the index_ member. + // fwd_iterator increment since (end) iterator comparison is + // done solely through the index_ member. pos_.node = node.right; pos_.value_offset = 0; if ( un == 0 ) [[ unlikely ]] @@ -315,14 +315,16 @@ void bptree_base::swap( bptree_base & other ) noexcept #endif } +bptree_base::base_iterator bptree_base::make_iter( iter_pos const pos ) noexcept { return { nodes_, pos }; } + [[ gnu::pure ]] bptree_base::iter_pos bptree_base::begin_pos() const noexcept { return { this->first_leaf(), 0 }; } [[ gnu::pure ]] bptree_base::iter_pos bptree_base:: end_pos() const noexcept { auto const last_leaf{ hdr().last_leaf_ }; return { last_leaf, node( last_leaf ).num_vals }; } -[[ gnu::pure ]] bptree_base::base_iterator bptree_base::begin() noexcept { return { this->nodes_, begin_pos() }; } -[[ gnu::pure ]] bptree_base::base_iterator bptree_base::end () noexcept { return { this->nodes_, end_pos() }; } +[[ gnu::pure ]] bptree_base::base_iterator bptree_base::begin() noexcept { return make_iter( begin_pos() ); } +[[ gnu::pure ]] bptree_base::base_iterator bptree_base::end () noexcept { return make_iter( end_pos() ); } [[ gnu::pure ]] bptree_base::base_random_access_iterator bptree_base::ra_begin() noexcept { return { *this, begin_pos(), 0 }; } [[ gnu::pure ]] bptree_base::base_random_access_iterator bptree_base::ra_end () noexcept { return { *this, end_pos(), size() }; } diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 41e61b5..3544266 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -70,6 +70,10 @@ TEST( bp_tree, playground ) auto const hint42{ bpt.lower_bound( 42 ) }; EXPECT_EQ( *hint42, 42 + 1 ); EXPECT_EQ( *bpt.insert( hint42, 42 ), 42 ); + + EXPECT_EQ( *bpt.erase( bpt.find( 42 ) ), 43 ); + EXPECT_TRUE( bpt.insert( 42 ).second ); + { auto const ra{ bpt.random_access() }; for ( auto n : std::views::iota( 0, test_size / 55 ) ) // slow operation (not really amortized constant time): use a smaller subset of the input From c4bf85f0b530f6664cdb5f3ed7087678ca96ba28 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Wed, 23 Oct 2024 23:42:40 +0200 Subject: [PATCH 45/49] Fixed node deallocation logic sometimes missing to update the leaf-list end 'pointers'. Moved more leaf node deallocation bits up into base classes. Fixed end() to work with empty containers. Fixed bptree_base_wkey::erase() handling one more edge case when deleting a value at the end of a leaf. Fixed erase( iterator ) to update separator keys. --- include/psi/vm/containers/b+tree.hpp | 49 ++++++++++++++++------------ src/containers/b+tree.cpp | 35 ++++++++++++++++++-- test/b+tree.cpp | 7 ++-- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index f844967..7ef4a21 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -283,7 +283,8 @@ class bptree_base [[ gnu::pure ]] depth_t leaf_level( ) const noexcept; [[ gnu::pure ]] bool is_leaf_level( depth_t const level ) const noexcept; - void free( node_header & ) noexcept; + void free ( node_header & ) noexcept; + void free_leaf( node_header & ) noexcept; void reserve_additional( node_slot::value_type additional_nodes ); void reserve ( node_slot::value_type new_capacity_in_number_of_nodes ); @@ -364,7 +365,8 @@ class bptree_base void rshift_sibling_parent_pos( node_header & node ) noexcept; void update_right_sibling_link( node_header const & left_node, node_slot left_node_slot ) noexcept; - void unlink_node( node_header & node, node_header & cached_left_sibling ) noexcept; + void unlink_and_free_node( node_header & node, node_header & cached_left_sibling ) noexcept; + void unlink_and_free_leaf( node_header & leaf, node_header & cached_left_sibling ) noexcept; void unlink_left ( node_header & nd ) noexcept; void unlink_right( node_header & nd ) noexcept; @@ -384,8 +386,8 @@ class bptree_base template static NodeType const & as( SourceNode const & slot ) noexcept { return as( const_cast( slot ) ); } - node_placeholder & node( node_slot const offset ) noexcept { return nodes_[ *offset ]; } - node_placeholder const & node( node_slot const offset ) const noexcept { return nodes_[ *offset ]; } + [[ gnu::pure ]] node_placeholder & node( node_slot const offset ) noexcept { return nodes_[ *offset ]; } + [[ gnu::pure ]] node_placeholder const & node( node_slot const offset ) const noexcept { return nodes_[ *offset ]; } auto & root() noexcept { return node( hdr().root_ ); } auto const & root() const noexcept { return const_cast( *this ).root(); } @@ -416,6 +418,8 @@ class bptree_base void assign_nodes_to_free_pool( node_slot::value_type starting_node ) noexcept; + void update_leaf_list_ends( node_header & removed_leaf ) noexcept; + protected: node_pool nodes_; #ifndef NDEBUG @@ -783,8 +787,8 @@ class bptree_base_wkey : public bptree_base auto const mid{ N::min_values }; BOOST_ASSUME( node_to_split.num_vals == max ); auto [node_slot, new_slot]{ bptree_base::new_spillover_node_for( node_to_split ) }; - auto p_node { &node( node_slot ) }; - auto p_new_node{ &node( new_slot ) }; + auto p_node { &node( node_slot ) }; + auto p_new_node{ &node( new_slot ) }; verify( *p_node ); BOOST_ASSUME( p_node->num_vals == max ); BOOST_ASSERT @@ -863,12 +867,17 @@ class bptree_base_wkey : public bptree_base root_ = {}; free( leaf ); --depth_; - next_pos = bptree_base::end_pos(); + BOOST_ASSUME( depth_ == 0 ); + BOOST_ASSUME( hdr.size_ == 1 ); + BOOST_ASSUME( !hdr.first_leaf_ ); + BOOST_ASSUME( !hdr.last_leaf_ ); + next_pos = {}; // empty end_pos } } else { auto p_leaf{ &leaf }; + bool last_node_value_was_erased{ leaf_key_offset == p_leaf->num_vals }; if ( underflowed( leaf ) ) { BOOST_ASSUME( !leaf.is_root() ); @@ -877,9 +886,10 @@ class bptree_base_wkey : public bptree_base next_pos.value_offset += leaf_key_offset; p_leaf = &this->leaf( next_pos.node ); BOOST_ASSUME( next_pos.value_offset <= p_leaf->num_vals ); + last_node_value_was_erased = ( next_pos.value_offset == p_leaf->num_vals ); } - if ( leaf_key_offset == p_leaf->num_vals ) // the last value in a node was erased + if ( last_node_value_was_erased ) { if ( !p_leaf->right ) { next_pos = bptree_base::end_pos(); @@ -1240,16 +1250,10 @@ class bptree_base_wkey : public bptree_base root_node const & root() const noexcept { return const_cast( *this ).root(); } using bptree_base::free; - void free( leaf_node & leaf ) noexcept { - auto & first_leaf{ hdr().first_leaf_ }; - if ( first_leaf == slot_of( leaf ) ) - { - BOOST_ASSUME( !leaf.left ); - first_leaf = leaf.right; - unlink_right( leaf ); - } - bptree_base::free( static_cast( leaf ) ); - } + void free( leaf_node & leaf ) noexcept { bptree_base::free_leaf( leaf ); } + + using bptree_base::unlink_and_free_node; + void unlink_and_free_node( leaf_node & leaf, leaf_node & cached_left_sibling ) noexcept { bptree_base::unlink_and_free_leaf( leaf, cached_left_sibling ); } template [[ gnu::sysv_abi ]] static @@ -1301,7 +1305,7 @@ class bptree_base_wkey : public bptree_base BOOST_ASSUME( parent.num_vals ); parent.num_vals--; - unlink_node( right, left ); + unlink_and_free_node( right, left ); verify( left ); verify( parent ); } @@ -1325,7 +1329,7 @@ class bptree_base_wkey : public bptree_base left.num_vals += right.num_vals; BOOST_ASSUME( left.num_vals >= left.max_values - 1 ); BOOST_ASSUME( left.num_vals <= left.max_values ); verify( left ); - unlink_node( right, left ); + unlink_and_free_node( right, left ); lshift_keys ( parent, parent_key_idx ); lshift_chldrn( parent, parent_child_idx ); @@ -1457,7 +1461,10 @@ bptree_base_wkey::iterator bptree_base_wkey::erase( const_iterator const iter ) noexcept { auto const [node, key_offset]{ iter.base().pos() }; - return static_cast( make_iter( erase( leaf( node ), key_offset ) ) ); + auto & lf{ leaf( node ) }; + if ( key_offset == 0 ) [[ unlikely ]] + update_separator( lf, lf.keys[ 1 ] ); + return static_cast( make_iter( erase( lf, key_offset ) ) ); } template diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index ef86e12..dfdf7f2 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -107,7 +107,7 @@ void bptree_base::update_right_sibling_link( node_header const & left_node, node } } -void bptree_base::unlink_node( node_header & node, node_header & cached_left_sibling ) noexcept +void bptree_base::unlink_and_free_node( node_header & node, node_header & cached_left_sibling ) noexcept { auto & left { cached_left_sibling }; auto & right{ node }; @@ -118,6 +118,30 @@ void bptree_base::unlink_node( node_header & node, node_header & cached_left_sib free( node ); } +void bptree_base::update_leaf_list_ends( node_header & removed_leaf ) noexcept +{ + auto & first_leaf{ hdr().first_leaf_ }; + auto & last_leaf{ hdr().last_leaf_ }; + auto const slot{ slot_of( removed_leaf ) }; + if ( slot == first_leaf ) [[ unlikely ]] + { + BOOST_ASSUME( !removed_leaf.left ); + first_leaf = removed_leaf.right; + } + if ( slot == last_leaf ) [[ unlikely ]] + { + BOOST_ASSUME( !removed_leaf.right ); + last_leaf = removed_leaf.left; + } +} + +void bptree_base::unlink_and_free_leaf( node_header & leaf, node_header & cached_left_sibling ) noexcept +{ + update_leaf_list_ends( leaf ); + unlink_and_free_node( leaf, cached_left_sibling ); +} + + void bptree_base::unlink_left( node_header & nd ) noexcept { if ( !nd.left ) @@ -320,7 +344,7 @@ bptree_base::base_iterator bptree_base::make_iter( iter_pos const pos ) noexcept [[ gnu::pure ]] bptree_base::iter_pos bptree_base::begin_pos() const noexcept { return { this->first_leaf(), 0 }; } [[ gnu::pure ]] bptree_base::iter_pos bptree_base:: end_pos() const noexcept { auto const last_leaf{ hdr().last_leaf_ }; - return { last_leaf, node( last_leaf ).num_vals }; + return { last_leaf, last_leaf ? node( last_leaf ).num_vals : node_size_type{} }; } [[ gnu::pure ]] bptree_base::base_iterator bptree_base::begin() noexcept { return make_iter( begin_pos() ); } @@ -400,6 +424,7 @@ void bptree_base::free( node_header & node ) noexcept auto & free_list{ hdr.free_list_ }; auto & free_node{ static_cast( node ) }; auto const free_node_slot{ slot_of( free_node ) }; + BOOST_ASSUME( free_node_slot != hdr.last_leaf_ ); // should have been handled in the dedicated overload #ifndef NDEBUG if ( free_node.left ) BOOST_ASSERT( left( free_node ).right != free_node_slot ); @@ -422,6 +447,12 @@ void bptree_base::free( node_header & node ) noexcept ++hdr.free_node_count_; } +void bptree_base::free_leaf( node_header & leaf ) noexcept +{ + update_leaf_list_ends( leaf ); + free( leaf ); +} + //------------------------------------------------------------------------------ } // namespace psi::vm //------------------------------------------------------------------------------ diff --git a/test/b+tree.cpp b/test/b+tree.cpp index 3544266..af80e76 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -109,8 +109,11 @@ TEST( bp_tree, playground ) std::shuffle( numbers.begin(), numbers.end(), rng ); for ( auto const & n : numbers ) EXPECT_TRUE( bpt.erase( n ) ); - for ( auto const & n : merge_appendix ) - EXPECT_TRUE( bpt.erase( n ) ); + for ( auto const & n : merge_appendix ) { + // + auto const next_it{ bpt.erase( bpt.find( n ) ) }; + EXPECT_TRUE( ( next_it == bpt.end() ) || ( *next_it == n + 1 ) ); + } EXPECT_TRUE( bpt.empty() ); } From b82ddd261ad844cec837217ce02c52c15d5d9869 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 26 Oct 2024 17:13:08 +0200 Subject: [PATCH 46/49] Fixed empty() not to crash w/ vectors w/o mapped storage. Various minor stylistic cleanups. --- include/psi/vm/vector.hpp | 41 +++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/include/psi/vm/vector.hpp b/include/psi/vm/vector.hpp index e66200c..2dcb7c9 100644 --- a/include/psi/vm/vector.hpp +++ b/include/psi/vm/vector.hpp @@ -33,10 +33,7 @@ #include #include //------------------------------------------------------------------------------ -namespace psi -{ -//------------------------------------------------------------------------------ -namespace vm +namespace psi::vm { //------------------------------------------------------------------------------ @@ -55,7 +52,7 @@ class contiguous_container_storage_base contiguous_container_storage_base & operator=( contiguous_container_storage_base && ) = default; auto data() noexcept { BOOST_ASSERT_MSG( mapping_, "Paging file not attached" ); return std::assume_aligned( view_.data() ); } - auto data() const noexcept { return const_cast< contiguous_container_storage_base & >( *this ).data(); } + auto data() const noexcept { return const_cast( *this ).data(); } void unmap() noexcept { view_.unmap(); } @@ -353,7 +350,7 @@ bool constexpr is_trivially_moveable #ifdef __clang__ __is_trivially_relocatable( T ) || #endif - std::is_trivially_move_constructible_v< T > + std::is_trivially_move_constructible_v }; struct default_init_t{}; inline constexpr default_init_t default_init; @@ -418,7 +415,7 @@ class vector using const_pointer = T const *; using reference = T &; using const_reference = T const &; - using param_const_ref = std::conditional_t< std::is_trivial_v< T > && ( sizeof( T ) <= 2 * sizeof( void * ) ), T, const_reference >; + using param_const_ref = std::conditional_t< std::is_trivial_v && ( sizeof( T ) <= 2 * sizeof( void * ) ), T, const_reference >; using size_type = sz_t; using difference_type = std::make_signed_t; using iterator = typename span_t:: iterator; @@ -585,7 +582,7 @@ class vector //! Throws: Nothing. //! //! Complexity: Constant. - [[ nodiscard, gnu::pure ]] bool empty() const noexcept { return span().empty(); } + [[ nodiscard, gnu::pure ]] bool empty() const noexcept { return !storage_ || span().empty(); } //! Effects: Returns the number of the elements contained in the vector. //! @@ -599,7 +596,7 @@ class vector //! Throws: Nothing. //! //! Complexity: Constant. - [[ nodiscard ]] static constexpr size_type max_size() noexcept { return static_cast< size_type >( std::numeric_limits< size_type >::max() / sizeof( value_type ) ); } + [[ nodiscard ]] static constexpr size_type max_size() noexcept { return static_cast( std::numeric_limits< size_type >::max() / sizeof( value_type ) ); } //! Effects: Inserts or erases elements at the end such that //! the size becomes n. New elements can be default or value initialized. @@ -607,7 +604,7 @@ class vector //! Throws: If memory allocation throws, or T's copy/move or value initialization throws. //! //! Complexity: Linear to the difference between size() and new_size. - void resize( size_type const new_size, default_init_t ) requires( std::is_trivial_v< T > ) + void resize( size_type const new_size, default_init_t ) requires( std::is_trivial_v ) { storage_.resize( to_byte_sz( new_size ) ); } @@ -841,7 +838,7 @@ class vector //! //! Complexity: Constant. [[ nodiscard ]] T * data() noexcept { return to_t_ptr( storage_.data() ); } - [[ nodiscard ]] T const * data() const noexcept { return const_cast< vector & >( *this ).data(); } + [[ nodiscard ]] T const * data() const noexcept { return const_cast( *this ).data(); } [[ nodiscard ]] span_t span() noexcept { return { data(), size() }; } [[ nodiscard ]] const_span_t span() const noexcept { return { data(), size() }; } @@ -916,7 +913,7 @@ class vector //! T's copy/move constructor throws. //! //! Complexity: Amortized constant time. - void push_back( T && x ) requires( !std::is_trivial_v< T > ) { emplace_back( std::move( x ) ); } + void push_back( T && x ) requires( !std::is_trivial_v ) { emplace_back( std::move( x ) ); } //! Requires: position must be a valid iterator of *this. //! @@ -936,7 +933,7 @@ class vector //! //! Complexity: If position is end(), amortized constant time //! Linear time otherwise. - iterator insert( const_iterator const position, T && x ) requires( !std::is_trivial_v< T > ) { return emplace( position, std::move( x ) ); } + iterator insert( const_iterator const position, T && x ) requires( !std::is_trivial_v ) { return emplace( position, std::move( x ) ); } //! Requires: position must be a valid iterator of *this. //! @@ -1223,15 +1220,15 @@ class vector }; // class vector -template < typename T, typename sz_t, bool headerless > +template #if defined( _MSC_VER ) && !defined( NDEBUG ) // Boost.Container flat_set is bugged: tries to dereference end iterators - asserts at runtime with secure/checked STL/containers // https://github.com/boostorg/container/issues/261 -requires is_trivially_moveable< T > -class unchecked_vector : public vector< T, sz_t, headerless > +requires is_trivially_moveable +class unchecked_vector : public vector { public: - using base = vector< T, sz_t, headerless >; + using base = vector; using iterator = typename base:: pointer; using const_iterator = typename base::const_pointer; @@ -1292,14 +1289,12 @@ class unchecked_vector : public vector< T, sz_t, headerless > } }; #else -using unchecked_vector = vector< T, sz_t, headerless >; +using unchecked_vector = vector; #endif -template < typename Vector > -using unchecked = unchecked_vector< typename Vector::value_type, typename Vector::size_type, Vector::headerless >; +template +using unchecked = unchecked_vector; //------------------------------------------------------------------------------ -} // namespace vm -//------------------------------------------------------------------------------ -} // namespace psi +} // namespace psi::vm //------------------------------------------------------------------------------ From 9f5f994cd39d215d5daa409f1558642c7e71c3ec Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 26 Oct 2024 18:57:38 +0200 Subject: [PATCH 47/49] Added the bp_tree::equal_range() public method. Using (faster) pdqsort from Boost.Move for bulk inserts. Using automatic minimal-size deduction for node_size_type. Optimized the core find() function. Various fixes around the pass_in_reg machinery. Made use_linear_search_for_sorted_array actually user specializable by converting it into a variable. Switched to using a smaller node size (with a brief comment). Fixed bulk_insert_prepare() for empty input. Fixed bulk_insert_into_empty() handling of short input sequences. Fixed node_count_required_for_values() logic. Fixed handling of an edge case in the bulk insert helper merge() function. Added basic benchmarking. --- CMakeLists.txt | 7 + include/psi/vm/containers/b+tree.hpp | 264 ++++++++++++++++++--------- src/containers/b+tree.cpp | 23 ++- test/b+tree.cpp | 78 +++++++- 4 files changed, 283 insertions(+), 89 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 226bfce..2ce1327 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,11 +23,15 @@ set( boost_ver boost-1.86.0 ) CPMAddPackage( "gh:boostorg/static_assert#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/throw_exception#${boost_ver}" ) # Boost::core dependency CPMAddPackage( "gh:boostorg/config#${boost_ver}" ) # Boost::core dependency +CPMAddPackage( "gh:boostorg/intrusive#${boost_ver}" ) # Boost::container dependency CPMAddPackage( "gh:boostorg/io#${boost_ver}" ) # Boost::utility dependency CPMAddPackage( "gh:boostorg/type_traits#${boost_ver}" ) # Boost::utility dependency CPMAddPackage( "gh:boostorg/predef#${boost_ver}" ) # Boost::winapi dependency CPMAddPackage( "gh:boostorg/assert#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/container#${boost_ver}" ) # used only for comparative benchmarking CPMAddPackage( "gh:boostorg/core#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/integer#${boost_ver}" ) +CPMAddPackage( "gh:boostorg/move#${boost_ver}" ) CPMAddPackage( "gh:boostorg/preprocessor#${boost_ver}" ) CPMAddPackage( "gh:boostorg/stl_interfaces#${boost_ver}" ) CPMAddPackage( "gh:boostorg/winapi#${boost_ver}" ) @@ -72,8 +76,11 @@ endif() include( vm.cmake ) target_link_libraries( psi_vm PUBLIC + Boost::container Boost::core Boost::assert + Boost::integer + Boost::move Boost::preprocessor Boost::stl_interfaces Boost::winapi diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 7ef4a21..010227c 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include @@ -39,10 +41,6 @@ namespace detail BOOST_ASSERT( remaining_space >= sizeof( Header ) ); return std::pair{ reinterpret_cast
( data ), std::span{ data + sizeof( Header ), remaining_space - sizeof( Header ) } }; } - - template static constexpr bool is_simple_comparator{ false }; - template static constexpr bool is_simple_comparator>{ true }; - template static constexpr bool is_simple_comparator>{ true }; } // namespace detail //////////////////////////////////////////////////////////////////////////////// @@ -88,7 +86,7 @@ struct [[ clang::trivial_abi ]] pass_in_reg static auto constexpr pass_by_val{ can_be_passed_in_reg }; using value_type = T; - using stored_type = std::conditional_t>; + using stored_type = std::conditional_t::type>; constexpr pass_in_reg( T const & u ) noexcept : val{ u } {} @@ -115,11 +113,14 @@ struct [[ clang::trivial_abi ]] pass_rv_in_reg }; // pass_rv_in_reg template -concept KeyType = ( transparent_comparator && std::is_convertible_v ) || std::is_same_v; +concept LookupType = transparent_comparator || std::is_same_v; + +template +concept InsertableType = ( transparent_comparator && std::is_convertible_v ) || std::is_same_v; -template bool constexpr reg { false }; -template bool constexpr reg>{ true }; -template bool constexpr reg>{ true }; +template bool constexpr reg { can_be_passed_in_reg }; +template bool constexpr reg>{ true }; +template bool constexpr reg>{ true }; template concept Reg = reg; @@ -128,23 +129,24 @@ concept Reg = reg; //////////////////////////////////////////////////////////////////////////////// -// user specializations and overloads of this function are allowed -template -consteval bool use_linear_search_for_sorted_array( [[ maybe_unused ]] std::uint32_t const minimum_array_length, std::uint32_t const maximum_array_length ) noexcept +// user specializations are allowed and intended: + +template constexpr bool is_simple_comparator{ false }; +template constexpr bool is_simple_comparator>{ true }; +template constexpr bool is_simple_comparator>{ true }; + +template constexpr bool is_statically_sized { true }; +template requires requires{ T{}.size(); } constexpr bool is_statically_sized{ T{}.size() != 0 }; + +template +constexpr bool use_linear_search_for_sorted_array { - auto const basic_test - { - detail::is_simple_comparator && - std::is_trivially_copyable_v && - sizeof( Key ) < ( 4 * sizeof( void * ) ) && - maximum_array_length < 2048 - }; - if constexpr ( requires{ Key{}.size(); } ) - { - return basic_test && ( Key{}.size() != 0 ); - } - return basic_test; -} + ( is_simple_comparator ) && + ( std::is_trivially_copyable_v ) && + ( sizeof( Key ) < ( 4 * sizeof( void * ) ) ) && + ( maximum_array_length * sizeof( Key ) <= 4096 ) && + ( is_statically_sized ) +}; // use_linear_search_for_sorted_array //////////////////////////////////////////////////////////////////////////////// @@ -171,6 +173,10 @@ class bptree_base storage_result map_file( auto const file, flags::named_object_construction_policy const policy ) noexcept { storage_result success{ nodes_.map_file( file, policy ) }; +# ifndef NDEBUG + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); +# endif if ( std::move( success ) && nodes_.empty() ) hdr() = {}; return success; @@ -178,7 +184,11 @@ class bptree_base storage_result map_memory( std::uint32_t initial_capacity_as_number_of_nodes = 0 ) noexcept; protected: +#if 0 // favoring CPU cache & branch prediction (linear scans) _vs_ TLB and disk access related issues, TODO make this configurable static constexpr auto node_size{ page_size }; +#else + static constexpr auto node_size{ 256 }; +#endif using depth_t = std::uint8_t; @@ -198,7 +208,8 @@ class bptree_base struct [[ nodiscard, clang::trivial_abi ]] node_header { - using size_type = std::uint16_t; + static auto constexpr minimum_header_size{ 3 * sizeof( node_slot ) + 2 * sizeof( /*minimum size_type*/ std::uint8_t ) }; + using size_type = typename boost::uint_value_t::least; // At minimum we need a single-linked/directed list in the vertical/depth // and horizontal/breadth directions (and the latter only for the leaf @@ -241,6 +252,7 @@ class bptree_base node_slot node {}; node_size_type value_offset{}; }; + class base_iterator; class base_random_access_iterator; @@ -263,6 +275,8 @@ class bptree_base void swap( bptree_base & other ) noexcept; base_iterator make_iter( iter_pos ) noexcept; + base_iterator make_iter( node_slot, node_size_type offset ) noexcept; + base_iterator make_iter( node_header const &, node_size_type offset ) noexcept; [[ gnu::pure ]] iter_pos begin_pos() const noexcept; [[ gnu::pure ]] iter_pos end_pos() const noexcept; @@ -301,10 +315,10 @@ class bptree_base //BOOST_ASSUME( node.num_vals >= node.min_values ); } - static constexpr auto keys ( auto & node ) noexcept { verify( node ); return std::span{ node.keys , node.num_vals }; } - static constexpr auto keys ( auto const & node ) noexcept { verify( node ); return std::span{ node.keys , node.num_vals }; } - static constexpr auto children( auto & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } - static constexpr auto children( auto const & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, node.num_vals + 1U }; else return std::array{}; } + static constexpr auto keys ( auto & node ) noexcept { verify( node ); return std::span{ node.keys , static_cast( node.num_vals ) }; } + static constexpr auto keys ( auto const & node ) noexcept { verify( node ); return std::span{ node.keys , static_cast( node.num_vals ) }; } + static constexpr auto children( auto & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, static_cast( node.num_vals + 1U ) }; else return std::array{}; } + static constexpr auto children( auto const & node ) noexcept { verify( node ); if constexpr ( requires{ node.children; } ) return std::span{ node.children, static_cast( node.num_vals + 1U ) }; else return std::array{}; } [[ gnu::pure ]] static constexpr node_size_type num_vals ( auto const & node ) noexcept { return node.num_vals; } [[ gnu::pure ]] static constexpr node_size_type num_chldrn( auto const & node ) noexcept { if constexpr ( requires{ node.children; } ) { BOOST_ASSUME( node.num_vals ); return node.num_vals + 1U; } else return 0; } @@ -422,8 +436,9 @@ class bptree_base protected: node_pool nodes_; -#ifndef NDEBUG - header const * hdr_{}; +#ifndef NDEBUG // debugging helpers (undoing type erasure done by contiguous_container_storage_base) + header const * p_hdr_ {}; + node_placeholder const * p_nodes_{}; #endif }; // class bptree_base @@ -559,7 +574,7 @@ class bptree_base_wkey : public bptree_base std::uint8_t max_inner_node_count{ depth > 1 }; for ( auto d{ 3 }; d < depth; ++d ) { - max_inner_node_count += max_inner_node_count * inner_node::max_children; + max_inner_node_count += static_cast( max_inner_node_count * inner_node::max_children ); } BOOST_ASSUME( max_inner_node_count < n ); return ( n - max_inner_node_count ) * leaf_node::max_values; @@ -596,7 +611,7 @@ class bptree_base_wkey : public bptree_base Key keys [ max_values ]; node_slot children[ max_children ]; - }; // struct inner_node + }; // struct parent_node struct inner_node : parent_node { @@ -825,6 +840,8 @@ class bptree_base_wkey : public bptree_base protected: // 'other' + [[ gnu::pure, nodiscard ]] auto make_iter( auto const &... args ) noexcept { return static_cast( bptree_base::make_iter( args... ) ); } + template insert_pos_t insert( N & target_node, node_size_type const target_node_pos, key_rv_arg v, node_slot const right_child ) { @@ -911,7 +928,7 @@ class bptree_base_wkey : public bptree_base // maintaining that invariant (TODO make this an option). bool bulk_append_fill_incomplete_leaf( leaf_node & leaf ) noexcept { - auto const missing_keys{ static_cast( std::max( 0, signed( leaf.min_values ) - leaf.num_vals ) ) }; + auto const missing_keys{ static_cast( std::max( 0, signed( leaf.min_values - leaf.num_vals ) ) ) }; if ( missing_keys ) { auto & preceding{ left( leaf ) }; @@ -968,6 +985,9 @@ class bptree_base_wkey : public bptree_base bulk_copied_input bulk_insert_prepare( std::ranges::subrange keys ) { + if ( keys.empty() ) [[ unlikely ]] + return bulk_copied_input{}; + auto constexpr can_preallocate{ kind == std::ranges::subrange_kind::sized }; if constexpr ( can_preallocate ) reserve_additional( static_cast( keys.size() ) ); @@ -980,7 +1000,7 @@ class bptree_base_wkey : public bptree_base size_type count{ 0 }; for ( ;; ) { - leaf_node & leaf{ this->leaf( leaf_slot ) }; + auto & leaf{ this->leaf( leaf_slot ) }; BOOST_ASSUME( leaf.num_vals == 0 ); if constexpr ( can_preallocate ) { auto const size_to_copy{ static_cast( std::min( leaf.max_values, static_cast( keys.end() - p_keys ) ) ) }; @@ -995,6 +1015,7 @@ class bptree_base_wkey : public bptree_base } count += leaf.num_vals; } + BOOST_ASSUME( leaf.num_vals > 0 ); --this->hdr().free_node_count_; if ( p_keys != keys.end() ) { @@ -1027,9 +1048,12 @@ class bptree_base_wkey : public bptree_base auto * hdr{ &this->hdr() }; hdr->root_ = begin_leaf; hdr->first_leaf_ = begin_leaf; + BOOST_ASSUME( hdr->depth_ == 0 ); if ( begin_leaf == end_leaf.node ) [[ unlikely ]] // single-node-sized initial insert { hdr->last_leaf_ = end_leaf.node; + hdr->size_ = total_size; + hdr->depth_ = ( total_size != 0 ); return; } auto const & first_root_left { leaf ( begin_leaf ) }; @@ -1040,8 +1064,13 @@ class bptree_base_wkey : public bptree_base new_root( begin_leaf, first_root_left.right, key_rv_arg{ /*mrmlj*/Key{ first_root_right.keys[ 0 ] } } ); // may invalidate references hdr = &this->hdr(); BOOST_ASSUME( hdr->depth_ == 2 ); - bulk_append( &leaf( first_unconnected_node ), { hdr->root_, 1 } ); - BOOST_ASSUME( hdr->last_leaf_ == end_leaf.node ); + if ( first_unconnected_node ) { // first check if there are more than two nodes + bulk_append( &leaf( first_unconnected_node ), { hdr->root_, 1 } ); + BOOST_ASSUME( hdr->last_leaf_ == end_leaf.node ); + } else { + hdr->last_leaf_ = end_leaf.node; + BOOST_ASSUME( !!hdr->last_leaf_ ); + } hdr->size_ = total_size; } @@ -1337,27 +1366,36 @@ class bptree_base_wkey : public bptree_base parent.num_vals--; } + static void verify( auto const & node ) noexcept + { + BOOST_ASSERT( std::ranges::adjacent_find( keys( node ) ) == keys( node ).end() ); + bptree_base::verify( node ); + } + private: [[ gnu::const, gnu::noinline ]] static node_slot::value_type node_count_required_for_values( size_type const number_of_values ) noexcept { - if ( !number_of_values ) - return 0; + if ( number_of_values <= leaf_node::max_values ) + return ( number_of_values != 0 ); auto const leaf_count{ static_cast( divide_up( number_of_values, /*assuming an 'optimistic' reserve, i.e. for bulk insert*/leaf_node::max_values ) ) }; - auto total_count{ node_slot::value_type{ 0 } }; + auto total_count{ leaf_count }; auto current_level_count{ leaf_count }; auto depth{ 1 }; while ( current_level_count > 1 ) { - total_count += current_level_count; current_level_count = divide_up( current_level_count, inner_node::min_children ); // pessimistic about inner node utilization + total_count += current_level_count; ++depth; } - // theoretical (+1 since we use a 1-based depth index) - auto const minimum_height{ static_cast( 1 + std::ceil( std::log( number_of_values + 1 ) / std::log( inner_node::max_children ) ) - 1 ) }; - auto const maximum_height{ static_cast( 1 + std::log( ( number_of_values + 1 ) / 2 ) / std::log( inner_node::min_children ) ) }; + // +1 since we use a 1-based depth index (instead of a 0-based where -1 + // is used to denote an empty tree) + auto const minimum_height{ static_cast( 1 + std::ceil( std::log( leaf_count ) / std::log( inner_node::max_children ) ) ) }; + auto const maximum_height{ static_cast( 1 + std::ceil( std::log( leaf_count ) / std::log( inner_node::min_children ) ) ) }; BOOST_ASSUME( depth >= minimum_height ); BOOST_ASSUME( depth <= maximum_height ); + [[ maybe_unused ]] + auto tree_structure_overhead{ total_count - leaf_count }; return total_count; } }; // class bptree_base_wkey @@ -1464,7 +1502,7 @@ bptree_base_wkey::erase( const_iterator const iter ) noexcept auto & lf{ leaf( node ) }; if ( key_offset == 0 ) [[ unlikely ]] update_separator( lf, lf.keys[ 1 ] ); - return static_cast( make_iter( erase( lf, key_offset ) ) ); + return make_iter( erase( lf, key_offset ) ); } template @@ -1570,6 +1608,9 @@ class bp_tree using iterator = base:: iterator; using const_iterator = base::const_iterator; + using iter_pair = std::pair< iterator, iterator>; + using const_iter_pair = std::pair; + using base::empty; using base::size; using base::clear; @@ -1598,15 +1639,16 @@ class bp_tree auto random_access() noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } auto random_access() const noexcept { return std::ranges::subrange{ ra_begin(), ra_end(), size() }; } - [[ nodiscard ]] bool contains ( KeyType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key } ); } - [[ nodiscard ]] iterator find ( KeyType auto const & key ) noexcept { return find_impl ( pass_in_reg{ key } ); } - [[ nodiscard ]] const_iterator find ( KeyType auto const & key ) const noexcept { return const_cast( *this ).find( key ); } - [[ nodiscard ]] iterator lower_bound( KeyType auto const & key ) noexcept { return lower_bound_impl( pass_in_reg{ key } ); } - [[ nodiscard ]] const_iterator lower_bound( KeyType auto const & key ) const noexcept { return const_cast( *this ).lower_bound( key ); } - - std::pair insert( KeyType auto const & key ) { return insert_impl( pass_in_reg{ key } ); } + [[ nodiscard ]] bool contains ( LookupType auto const & key ) const noexcept { return contains_impl ( pass_in_reg{ key } ); } + [[ nodiscard ]] iterator find ( LookupType auto const & key ) noexcept { return find_impl ( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iterator find ( LookupType auto const & key ) const noexcept { return const_cast( *this ).find( key ); } + [[ nodiscard ]] iterator lower_bound( LookupType auto const & key ) noexcept { return lower_bound_impl( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iterator lower_bound( LookupType auto const & key ) const noexcept { return const_cast( *this ).lower_bound( key ); } + [[ nodiscard ]] iter_pair equal_range( LookupType auto const & key ) noexcept { return equal_range_impl( pass_in_reg{ key } ); } + [[ nodiscard ]] const_iter_pair equal_range( LookupType auto const & key ) const noexcept { return const_cast( *this ).equal_range( key ); } - iterator insert( const_iterator const pos_hint, KeyType auto const & key ) { return insert_impl( pos_hint, pass_in_reg{ key } ); } + std::pair insert( InsertableType auto const & key ) { return insert_impl( pass_in_reg{ key } ); } + iterator insert( const_iterator const pos_hint, InsertableType auto const & key ) { return insert_impl( pos_hint, pass_in_reg{ key } ); } // bulk insert // performance note: insertion of existing values into a unique bp_tree is @@ -1661,7 +1703,7 @@ class bp_tree private: // pass-in-reg public function overloads/impls bool contains_impl( Reg auto const key ) const noexcept { return !empty() && const_cast( *this ).find_nodes_for( key ).leaf_offset.exact_find; } - [[ using gnu: noinline, pure, sysv_abi ]] + [[ using gnu: pure, sysv_abi ]] iterator find_impl( Reg auto const key ) noexcept { if ( !empty() ) [[ likely ]] @@ -1680,12 +1722,28 @@ class bp_tree if ( !empty() ) [[ likely ]] { auto const location{ find_nodes_for( key ) }; - return iterator{ this->nodes_, { slot_of( location.leaf ), location.leaf_offset.pos } }; + return base::make_iter( location.leaf, location.leaf_offset.pos ); } return this->end(); } + [[ using gnu: pure, sysv_abi ]] + iter_pair equal_range_impl( Reg auto const key ) noexcept + { + if ( !empty() ) [[ likely ]] + { + auto const location{ find_nodes_for( key ) }; + if ( location.leaf_offset.exact_find ) [[ likely ]] { + auto const begin{ base::make_iter( location.leaf, location.leaf_offset.pos ) }; + return { begin, std::next( begin ) }; + } + } + + auto const end_iter{ end() }; + return { end_iter, end_iter }; + } + std::pair insert_impl( Reg auto const v ) { if ( empty() ) @@ -1700,11 +1758,11 @@ class bp_tree BOOST_ASSUME( !locations.inner ); BOOST_ASSUME( !locations.inner_offset ); if ( locations.leaf_offset.exact_find ) [[ unlikely ]] - return { { this->nodes_, { slot_of( locations.leaf ), locations.leaf_offset.pos } }, false }; + return { base::make_iter( locations.leaf, locations.leaf_offset.pos ), false }; auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; ++this->hdr().size_; - return { std::prev( iterator{ this->nodes_, { insert_pos_next.node, insert_pos_next.next_insert_offset } } ), true }; + return { std::prev( base::make_iter( insert_pos_next.node, insert_pos_next.next_insert_offset ) ), true }; } iterator insert_impl( const_iterator const pos_hint, Reg auto const v ) @@ -1717,38 +1775,66 @@ class bp_tree auto const [hint_slot, hint_slot_offset]{ pos_hint.base().pos() }; auto const insert_pos_next{ base::insert( leaf( hint_slot ), hint_slot_offset, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; ++this->hdr().size_; - return std::prev( iterator{ this->nodes_, { insert_pos_next.node, insert_pos_next.next_insert_offset } } ); + return std::prev( base::make_iter( insert_pos_next.node, insert_pos_next.next_insert_offset ) ); } private: - // lower_bound find - struct find_pos // msvc pass-in-reg facepalm + // key_locations (containing find_pos) has to returnable through registers + struct find_pos0 // msvc pass-in-reg facepalm { node_size_type pos : ( sizeof( node_size_type ) * CHAR_BIT - 1 ); node_size_type exact_find : 1; }; - [[ using gnu: pure, hot, noinline, sysv_abi ]] - find_pos find( Key const keys[], node_size_type const num_vals, Reg auto const value ) const noexcept + struct find_pos1 { - // TODO branchless binary search, Alexandrescu's ideas, https://orlp.net/blog/bitwise-binary-search ... - BOOST_ASSUME( num_vals > 0 ); - auto const & __restrict comp{ this->comp() }; - node_size_type pos_idx; - if constexpr ( use_linear_search_for_sorted_array( 1, leaf_node::max_values ) ) + node_size_type pos; + bool exact_find; + }; + using find_pos = std::conditional_t + < + ( sizeof( find_pos1 ) > 2 ) && + ( leaf_node::max_values <= ( std::numeric_limits::max() / 2 ) ), + find_pos0, + find_pos1 + >; + + // lower_bound find + [[ using gnu: pure, hot, noinline, sysv_abi ]] + static find_pos find( Key const keys[], node_size_type const num_vals, Reg auto const value, pass_in_reg const comparator ) noexcept + { + // TODO branchless binary search, Alexandrescu's TLC, + // https://orlp.net/blog/bitwise-binary-search + // https://algorithmica.org/en/eytzinger + // FAST: Fast Architecture Sensitive Tree Search on Modern CPUs and GPUs http://kaldewey.com/pubs/FAST__SIGMOD10.pdf + // ... + BOOST_ASSUME( num_vals > 0 ); + BOOST_ASSUME( num_vals <= leaf_node::max_values ); + Comparator const & __restrict comp( comparator ); + if constexpr ( use_linear_search_for_sorted_array ) { - auto k{ 0 }; - while ( ( k != num_vals ) && comp( keys[ k ], value ) ) - ++k; - pos_idx = static_cast( k ); + for ( node_size_type k{ 0 }; ; ) + { + if ( !comp( keys[ k ], value ) ) + { + auto const exact_find{ !comp( value, keys[ k ] ) }; + return { k, exact_find }; + } + if ( ++k == num_vals ) + { + return { k, false }; + } + } } else { - auto const pos_iter{ std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp ) }; - pos_idx = static_cast( std::distance( &keys[ 0 ], pos_iter ) ); + auto const pos_iter { std::lower_bound( &keys[ 0 ], &keys[ num_vals ], value, comp ) }; + auto const pos_idx { static_cast( std::distance( &keys[ 0 ], pos_iter ) ) }; + auto const exact_find{ ( pos_idx != num_vals ) && !comp( value, keys[ pos_idx ] ) }; + return { pos_idx, exact_find }; } - auto const exact_find{ ( pos_idx != num_vals ) & !comp( value, keys[ std::min( pos_idx, num_vals - 1 ) ] ) }; - return { pos_idx, reinterpret_cast( exact_find ) }; + std::unreachable(); } + find_pos find( Key const keys[], node_size_type const num_vals, Reg auto const value ) const noexcept { return find( keys, num_vals, value, pass_in_reg{ comp() } ); } find_pos find( auto const & node, auto const & value ) const noexcept { return find( node.keys, node.num_vals, pass_in_reg{ value } ); } [[ using gnu: pure, hot, sysv_abi ]] find_pos find_with_offset( auto const & node, node_size_type const offset, Reg auto const value ) const noexcept @@ -1768,7 +1854,7 @@ class bp_tree node_slot inner; }; - [[ using gnu: pure, hot, sysv_abi ]] + [[ using gnu: pure, hot, sysv_abi, noinline ]] key_locations find_nodes_for( Reg auto const key ) noexcept { node_slot separator_key_node; @@ -1776,7 +1862,7 @@ class bp_tree // a leaf (lone) root is implicitly handled by the loop condition: // depth_ == 1 so the loop is skipped entirely and the lone root is // never examined through the incorrectly typed reference - auto p_node{ &bptree_base::as( root() ) }; + auto p_node{ &as( root() ) }; auto const depth { this->hdr().depth_ }; BOOST_ASSUME( depth >= 1 ); for ( auto level{ 0 }; level < depth - 1; ++level ) @@ -1801,7 +1887,7 @@ class bp_tree separator_key_node }; } - key_locations find_nodes_for( Key const & key ) noexcept { return find_nodes_for( pass_in_reg{ key } ); } + key_locations find_nodes_for( Key const & key ) noexcept { return find_nodes_for( key ); } auto find_next( leaf_node const & starting_leaf, node_size_type const starting_leaf_offset, Reg auto const key ) const noexcept { @@ -1920,7 +2006,7 @@ class bp_tree { if constexpr ( requires{ comp().eq( left, right ); } ) return comp().eq( left, right ); - if constexpr ( detail::is_simple_comparator && requires{ left == right; } ) + if constexpr ( is_simple_comparator && requires{ left == right; } ) return left == right; return !comp()( left, right ) && !comp()( right, left ); } @@ -1982,9 +2068,14 @@ auto bp_tree::merge // position in the target node that immediately follows the // position for the inserted src_keys[ 0 ] - IOW it need not be // the position for src.keys[ next_src_offset ] - if ( next_tgt_offset != tgt.num_vals ) // necessary check because find assumes non-empty input + if + ( + ( next_tgt_offset != tgt.num_vals ) && // necessary check because find assumes non-empty input + ( next_src_offset != src.num_vals ) && // possible edge case where there was actually only one key left in the source + false // not really worth it: the caller still has to call find on/for returns from all the other branches + ) { - next_tgt_offset = find_with_offset( tgt, next_tgt_offset, pass_in_reg{ src.keys[ next_src_offset ] } ).pos; + next_tgt_offset = find_with_offset( tgt, next_tgt_offset, key_const_arg{ src.keys[ next_src_offset ] } ).pos; } return std::make_tuple( 1, 1, &tgt, next_tgt_offset ); } @@ -1996,7 +2087,7 @@ auto bp_tree::merge if ( target.right ) { auto const & right_delimiter { right( target ).keys[ 0 ] }; - auto const less_than_right_pos{ find( src_keys, copy_size, pass_in_reg{ right_delimiter } ) }; + auto const less_than_right_pos{ find( src_keys, copy_size, key_const_arg{ right_delimiter } ) }; BOOST_ASSUME( !less_than_right_pos.exact_find ); if ( less_than_right_pos.pos != copy_size ) { @@ -2051,11 +2142,16 @@ bp_tree::insert( typename base::bulk_copied_input const input ) auto const [begin_leaf, end_pos, total_size]{ input }; ra_iterator const p_new_nodes_begin{ *this, { begin_leaf, 0 }, 0 }; ra_iterator const p_new_nodes_end { *this, end_pos , total_size }; +#if 0 // slower std::sort( p_new_nodes_begin, p_new_nodes_end, comp() ); +#else + boost::movelib::pdqsort( p_new_nodes_begin, p_new_nodes_end, comp() ); +#endif if ( empty() ) { base::bulk_insert_into_empty( begin_leaf, end_pos, total_size ); + BOOST_ASSUME( this->hdr().size_ == total_size ); return total_size; } @@ -2132,7 +2228,7 @@ bp_tree::insert( typename base::bulk_copied_input const input ) // that we are using presorted data) rather than starting everytime from // scratch (using find_nodes_for) std::tie( tgt_leaf, tgt_leaf_next_pos ) = - find_next( *tgt_leaf, tgt_next_offset, pass_in_reg{ src_leaf->keys[ source_slot_offset ] } ); + find_next( *tgt_leaf, tgt_next_offset, key_const_arg{ src_leaf->keys[ source_slot_offset ] } ); } while ( p_new_keys != p_new_nodes_end ); BOOST_ASSUME( inserted <= total_size ); diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index dfdf7f2..8aa603a 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -5,9 +5,11 @@ namespace psi::vm //------------------------------------------------------------------------------ // https://en.wikipedia.org/wiki/B-tree +// https://en.wikipedia.org/wiki/B%2B_tree // https://opendsa-server.cs.vt.edu/ODSA/Books/CS3/html/BTree.html // http://www.cburch.com/cs/340/reading/btree/index.html // https://www.programiz.com/dsa/b-plus-tree +// https://courses.cs.washington.edu/courses/cse332/23su/lectures/9_B_Trees.pdf (version, as this impl, which has different key counts in inner vs leaf nodes) // https://www.geeksforgeeks.org/b-trees-implementation-in-c // https://github.com/jeffplaisance/BppTree // https://flatcap.github.io/linux-ntfs/ntfs/concepts/tree/index.html @@ -17,6 +19,8 @@ namespace psi::vm // https://www.researchgate.net/publication/220225482_Cache-Oblivious_Databases_Limitations_and_Opportunities // https://www.postgresql.org/docs/current/btree.html // https://abseil.io/about/design/btree +// https://www.scylladb.com/2021/11/23/the-taming-of-the-b-trees +// https://www.scattered-thoughts.net/writing/smolderingly-fast-btrees // Data Structure Visualizations https://www.cs.usfca.edu/~galles/visualization/Algorithms.html // Griffin: Fast Transactional Database Index with Hash and B+-Tree https://ieeexplore.ieee.org/abstract/document/10678674 // Restructuring the concurrent B+-tree with non-blocked search operations https://www.sciencedirect.com/science/article/abs/pii/S002002550200261X @@ -37,6 +41,7 @@ void bptree_base::clear() noexcept std::span bptree_base::user_header_data() noexcept { return header_data().second; } +[[ gnu::pure, gnu::hot, gnu::always_inline ]] bptree_base::header & bptree_base::hdr() noexcept { return *header_data().first; } @@ -44,6 +49,10 @@ bptree_base::storage_result bptree_base::map_memory( std::uint32_t const initial_capacity_as_number_of_nodes ) noexcept { storage_result success{ nodes_.map_memory( initial_capacity_as_number_of_nodes, value_init ) }; +#ifndef NDEBUG + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); +#endif if ( std::move( success ) ) { hdr() = {}; @@ -61,7 +70,8 @@ void bptree_base::reserve_additional( node_slot::value_type additional_nodes ) auto const current_size{ nodes_.size() }; nodes_.grow_by( additional_nodes, value_init ); #ifndef NDEBUG - hdr_ = &hdr(); + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); #endif assign_nodes_to_free_pool( current_size ); } @@ -73,7 +83,8 @@ void bptree_base::reserve( node_slot::value_type new_capacity_in_number_of_nodes auto const current_size{ nodes_.size() }; nodes_.grow_to( new_capacity_in_number_of_nodes, value_init ); #ifndef NDEBUG - hdr_ = &hdr(); + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); #endif assign_nodes_to_free_pool( current_size ); } @@ -335,11 +346,14 @@ void bptree_base::swap( bptree_base & other ) noexcept using std::swap; swap( this->nodes_, other.nodes_ ); #ifndef NDEBUG - swap( this->hdr_, other.hdr_ ); + swap( this->p_hdr_ , other.p_hdr_ ); + swap( this->p_nodes_, other.p_nodes_ ); #endif } bptree_base::base_iterator bptree_base::make_iter( iter_pos const pos ) noexcept { return { nodes_, pos }; } +bptree_base::base_iterator bptree_base::make_iter( node_slot const node, node_size_type const offset ) noexcept { return make_iter({ node, offset }); } +bptree_base::base_iterator bptree_base::make_iter( node_header const & node, node_size_type const offset ) noexcept { return make_iter( slot_of( node ), offset ); } [[ gnu::pure ]] bptree_base::iter_pos bptree_base::begin_pos() const noexcept { return { this->first_leaf(), 0 }; } [[ gnu::pure ]] bptree_base::iter_pos bptree_base:: end_pos() const noexcept { @@ -413,7 +427,8 @@ bptree_base::new_node() BOOST_ASSUME( !new_node.left ); BOOST_ASSUME( !new_node.right ); #ifndef NDEBUG - hdr_ = &this->hdr(); + p_hdr_ = &this->hdr(); + p_nodes_ = nodes_.data(); #endif return new_node; } diff --git a/test/b+tree.cpp b/test/b+tree.cpp index af80e76..9d8c923 100644 --- a/test/b+tree.cpp +++ b/test/b+tree.cpp @@ -2,9 +2,17 @@ #include #include +#include + +#define HAVE_ABSL 0 +#if HAVE_ABSL +#include +#endif #include +#include +#include #include #include #include @@ -14,6 +22,72 @@ namespace psi::vm { //------------------------------------------------------------------------------ +#ifdef NDEBUG // bench only release builds + +namespace +{ + using timer = std::chrono::high_resolution_clock; + using duration = std::chrono::nanoseconds; + + duration time_insertion( auto & container, auto const & data ) + { + auto const start{ timer::now() }; + container.insert( data.begin(), data.end() ); + return std::chrono::duration_cast( timer::now() - start ) / data.size(); + } + duration time_lookup( auto const & container, auto const & data ) noexcept + { + auto const start{ timer::now() }; + for ( auto const x : data ) { + EXPECT_EQ( *container.find( x ), x ); + } + return std::chrono::duration_cast( timer::now() - start ) / data.size(); + } +} // anonymous namespace + +TEST( bp_tree, benchamrk ) +{ + auto const test_size{ 7654321 }; + auto const seed{ std::random_device{}() }; + std::mt19937 rng{ seed }; + + std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; + auto numbers{ std::ranges::to( sorted_numbers ) }; + std::ranges::shuffle( numbers, rng ); + + psi::vm ::bp_tree bpt; bpt.map_memory(); + boost::container::flat_set flat_set; +#if HAVE_ABSL + absl ::btree_set abpt; +#endif + + // bulk-insertion-into-empty + auto const flat_set_insert{ time_insertion( flat_set, sorted_numbers ) }; + auto const bpt_insert { time_insertion( bpt , sorted_numbers ) }; +#if HAVE_ABSL + auto const abpt_insert { time_insertion( abpt , sorted_numbers ) }; +#endif + + // random lookup + auto const flat_set_find{ time_lookup( flat_set, numbers ) }; + auto const bpt_find{ time_lookup( bpt , numbers ) }; +#if HAVE_ABSL + auto const abpt_find{ time_lookup( abpt , numbers ) }; +#endif + + std::println( "insert / lookup:" ); + std::println( "\t boost::container::flat_set:\t{} / {}", flat_set_insert, flat_set_find ); + std::println( "\t psi::vm::bpt:\t{} / {}", bpt_insert, bpt_find ); +#if HAVE_ABSL + std::println( "\t absl::bpt:\t{} / {}", abpt_insert, abpt_find ); +#endif + +#if 0 // CI servers are unreliable for comparative perf tests + EXPECT_LE( bpt_find, flat_set_find ); +#endif +} // bp_tree.benchamrk +#endif // release build + static auto const test_file{ "test.bpt" }; TEST( bp_tree, playground ) @@ -26,7 +100,9 @@ TEST( bp_tree, playground ) auto const test_size{ 258735 }; #endif std::ranges::iota_view constexpr sorted_numbers{ 0, test_size }; - std::mt19937 rng{ std::random_device{}() }; + auto const seed{ std::random_device{}() }; + std::println( "Seed {}", seed ); + std::mt19937 rng{ seed }; auto numbers{ std::ranges::to( sorted_numbers ) }; std::span const nums{ numbers }; // leave the largest quarter of values at the end to trigger/exercise the From ace0b4b1d8fc3a092c4217df78526d8c26fbd923 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Sat, 26 Oct 2024 23:28:17 +0200 Subject: [PATCH 48/49] Extracted more make_iter functionality into the base class(es). Fixed an edge case handling in lower_bound_impl(). Fixed hinted insert to handle the edge case requiring separator key update. --- include/psi/vm/containers/b+tree.hpp | 100 ++++++++++++++++----------- src/containers/b+tree.cpp | 8 ++- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 010227c..6cf7d58 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -258,6 +258,17 @@ class bptree_base struct insert_pos_t { node_slot node; node_size_type next_insert_offset; }; // TODO 'merge' with iter_pos + struct find_pos0 // msvc pass-in-reg facepalm + { + node_size_type pos : ( sizeof( node_size_type ) * CHAR_BIT - 1 ); + node_size_type exact_find : 1; + }; + struct find_pos1 + { + node_size_type pos; + bool exact_find; + }; + struct header // or persisted data members { node_slot root_; @@ -277,6 +288,7 @@ class bptree_base base_iterator make_iter( iter_pos ) noexcept; base_iterator make_iter( node_slot, node_size_type offset ) noexcept; base_iterator make_iter( node_header const &, node_size_type offset ) noexcept; + base_iterator make_iter( insert_pos_t ) noexcept; [[ gnu::pure ]] iter_pos begin_pos() const noexcept; [[ gnu::pure ]] iter_pos end_pos() const noexcept; @@ -840,7 +852,25 @@ class bptree_base_wkey : public bptree_base protected: // 'other' - [[ gnu::pure, nodiscard ]] auto make_iter( auto const &... args ) noexcept { return static_cast( bptree_base::make_iter( args... ) ); } + // key_locations (containing find_pos) should also be returnable through registers + using find_pos = std::conditional_t + < + ( sizeof( find_pos1 ) > 2 ) && + ( leaf_node::max_values <= ( std::numeric_limits::max() / 2 ) ), // we get only half the range if one bit is shaved off for exact_find + bptree_base::find_pos0, + bptree_base::find_pos1 + >; + struct key_locations + { + leaf_node & leaf; + find_pos leaf_offset; + // optional - if also present in an inner node as a separator key + node_size_type inner_offset; // ordered for compact layout + node_slot inner; + }; // struct key_locations + + [[ gnu::pure, nodiscard ]] iterator make_iter( auto const &... args ) noexcept { return static_cast( bptree_base::make_iter( args... ) ); } + [[ gnu::pure, nodiscard ]] iterator make_iter( key_locations const loc ) noexcept { return make_iter( loc.leaf, loc.leaf_offset.pos ); } template insert_pos_t insert( N & target_node, node_size_type const target_node_pos, key_rv_arg v, node_slot const right_child ) @@ -1571,6 +1601,7 @@ class bp_tree using parent_node = base::parent_node; using fwd_iterator = base::fwd_iterator; using ra_iterator = base:: ra_iterator; + using find_pos = base::find_pos; using iter_pos = base::iter_pos; using bptree_base::as; @@ -1710,22 +1741,31 @@ class bp_tree { auto const location{ find_nodes_for( key ) }; if ( location.leaf_offset.exact_find ) [[ likely ]] { - return iterator{ this->nodes_, { slot_of( location.leaf ), location.leaf_offset.pos } }; + return base::make_iter( location ); } } return this->end(); } - + [[ using gnu: pure, sysv_abi ]] iterator lower_bound_impl( Reg auto const key ) noexcept { if ( !empty() ) [[ likely ]] { auto const location{ find_nodes_for( key ) }; - return base::make_iter( location.leaf, location.leaf_offset.pos ); + // find_nodes_for() returns an insertion point, which can be one + // pass the end of a node (like an end iterator, indicating an + // append/push_back position) but iterators do not support this + // (have to be initialized with a valid position for the increment + // operator to work). + if ( location.leaf_offset.pos == location.leaf.num_vals ) [[ unlikely ]] { + BOOST_ASSUME( !location.leaf_offset.exact_find ); + return base::make_iter( location.leaf.right, node_size_type{ 0 } ); + } + return base::make_iter( location ); } - return this->end(); + return end(); } [[ using gnu: pure, sysv_abi ]] @@ -1735,7 +1775,7 @@ class bp_tree { auto const location{ find_nodes_for( key ) }; if ( location.leaf_offset.exact_find ) [[ likely ]] { - auto const begin{ base::make_iter( location.leaf, location.leaf_offset.pos ) }; + auto const begin{ base::make_iter( location ) }; return { begin, std::next( begin ) }; } } @@ -1758,46 +1798,33 @@ class bp_tree BOOST_ASSUME( !locations.inner ); BOOST_ASSUME( !locations.inner_offset ); if ( locations.leaf_offset.exact_find ) [[ unlikely ]] - return { base::make_iter( locations.leaf, locations.leaf_offset.pos ), false }; + return { base::make_iter( locations ), false }; auto const insert_pos_next{ base::insert( locations.leaf, locations.leaf_offset.pos, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; ++this->hdr().size_; - return { std::prev( base::make_iter( insert_pos_next.node, insert_pos_next.next_insert_offset ) ), true }; + return { base::make_iter( insert_pos_next ), true }; } iterator insert_impl( const_iterator const pos_hint, Reg auto const v ) { - // yes, for starters generic 'hint as just a hint' is not supported + // yes, for starters, generic 'hint as just a hint' is not supported BOOST_ASSUME( !empty() ); BOOST_ASSERT_MSG( le( v, *pos_hint ), "Invalid insertion hint" ); BOOST_ASSERT_MSG( !unique || ge( v, *std::prev( pos_hint ) ), "Invalid insertion hint" ); auto const [hint_slot, hint_slot_offset]{ pos_hint.base().pos() }; - auto const insert_pos_next{ base::insert( leaf( hint_slot ), hint_slot_offset, Key{ v }, { /*insertion starts from leaves which do not have children*/ } ) }; + auto & hint_leaf{ leaf( hint_slot ) }; + if ( hint_slot_offset == 0 ) [[ unlikely ]] { + base::update_separator( hint_leaf, v ); + } + [[ maybe_unused ]] + auto const insert_pos_next{ base::insert( hint_leaf, hint_slot_offset, Key{ v }, {} ) }; + BOOST_ASSERT( pos_hint == base::make_iter( insert_pos_next ) ); ++this->hdr().size_; - return std::prev( base::make_iter( insert_pos_next.node, insert_pos_next.next_insert_offset ) ); + return pos_hint.base(); } private: - // key_locations (containing find_pos) has to returnable through registers - struct find_pos0 // msvc pass-in-reg facepalm - { - node_size_type pos : ( sizeof( node_size_type ) * CHAR_BIT - 1 ); - node_size_type exact_find : 1; - }; - struct find_pos1 - { - node_size_type pos; - bool exact_find; - }; - using find_pos = std::conditional_t - < - ( sizeof( find_pos1 ) > 2 ) && - ( leaf_node::max_values <= ( std::numeric_limits::max() / 2 ) ), - find_pos0, - find_pos1 - >; - // lower_bound find [[ using gnu: pure, hot, noinline, sysv_abi ]] static find_pos find( Key const keys[], node_size_type const num_vals, Reg auto const value, pass_in_reg const comparator ) noexcept @@ -1845,17 +1872,8 @@ class bp_tree return result; } - struct key_locations - { - leaf_node & leaf; - find_pos leaf_offset; - // optional - if also present in an inner node as a separator key - node_size_type inner_offset; // ordered for compact layout - node_slot inner; - }; - [[ using gnu: pure, hot, sysv_abi, noinline ]] - key_locations find_nodes_for( Reg auto const key ) noexcept + base::key_locations find_nodes_for( Reg auto const key ) noexcept { node_slot separator_key_node; node_size_type separator_key_offset{}; @@ -1887,7 +1905,7 @@ class bp_tree separator_key_node }; } - key_locations find_nodes_for( Key const & key ) noexcept { return find_nodes_for( key ); } + auto find_nodes_for( Key const & key ) noexcept { return find_nodes_for( key ); } auto find_next( leaf_node const & starting_leaf, node_size_type const starting_leaf_offset, Reg auto const key ) const noexcept { diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index 8aa603a..efe91eb 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -352,8 +352,14 @@ void bptree_base::swap( bptree_base & other ) noexcept } bptree_base::base_iterator bptree_base::make_iter( iter_pos const pos ) noexcept { return { nodes_, pos }; } -bptree_base::base_iterator bptree_base::make_iter( node_slot const node, node_size_type const offset ) noexcept { return make_iter({ node, offset }); } +bptree_base::base_iterator bptree_base::make_iter( node_slot const node, node_size_type const offset ) noexcept { return make_iter(iter_pos{ node, offset }); } bptree_base::base_iterator bptree_base::make_iter( node_header const & node, node_size_type const offset ) noexcept { return make_iter( slot_of( node ), offset ); } +bptree_base::base_iterator bptree_base::make_iter( insert_pos_t const next_pos ) noexcept +{ + auto iter{ make_iter( next_pos.node, next_pos.next_insert_offset ) }; + --iter; + return iter; +} [[ gnu::pure ]] bptree_base::iter_pos bptree_base::begin_pos() const noexcept { return { this->first_leaf(), 0 }; } [[ gnu::pure ]] bptree_base::iter_pos bptree_base:: end_pos() const noexcept { From fedd814576930e88e7ff8d9733baaa13b4112c48 Mon Sep 17 00:00:00 2001 From: Domagoj Saric Date: Mon, 28 Oct 2024 08:57:04 +0100 Subject: [PATCH 49/49] Extracted pass_in_reg machinery to a dedicated header. Added more documentation links. Minor debug build bug fixes. --- include/psi/vm/containers/abi.hpp | 107 +++++++++++++++++++++++++++ include/psi/vm/containers/b+tree.hpp | 87 ++-------------------- src/containers/b+tree.cpp | 18 ++++- 3 files changed, 127 insertions(+), 85 deletions(-) create mode 100644 include/psi/vm/containers/abi.hpp diff --git a/include/psi/vm/containers/abi.hpp b/include/psi/vm/containers/abi.hpp new file mode 100644 index 0000000..3bc3d29 --- /dev/null +++ b/include/psi/vm/containers/abi.hpp @@ -0,0 +1,107 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +//------------------------------------------------------------------------------ +namespace psi::vm +{ +//------------------------------------------------------------------------------ + +PSI_WARNING_DISABLE_PUSH() +PSI_WARNING_MSVC_DISABLE( 5030 ) // unrecognized attribute + +//////////////////////////////////////////////////////////////////////////////// +// Modern(ized) attempt at 'automatized' boost::call_traits primarily to support +// efficient transparent comparators & non-inlined generic lookup functions +// which cause neither unnecessary copies of non-trivial types nor pass-by-ref +// of trivial ones. +// Largely still WiP... +// Essentially this is 'explicit IPA SROA'. +// https://gcc.gnu.org/onlinedocs/gccint/passes-and-files-of-the-compiler/inter-procedural-optimization-passes.html +//////////////////////////////////////////////////////////////////////////////// + +template +bool constexpr can_be_passed_in_reg +{ + ( + std::is_trivial_v && + ( sizeof( T ) <= 2 * sizeof( void * ) ) // assuming a sane ABI like SysV (ignoring the MS x64 disaster) + ) +#if defined( __GNUC__ ) || defined( __clang__ ) + || // detect SIMD types (this could also produce false positives for large compiler-native vectors that do not fit into the register file) + requires{ __builtin_convertvector( T{}, T ); } +#endif + // This is certainly not an exhaustive list/'trait' - certain types that can + // be passed in reg cannot be detected as such by existing compiler + // functionality, e.g. Homogeneous Vector Aggregates + // https://devblogs.microsoft.com/cppblog/introducing-vector-calling-convention + // users are encouraged to provide specializations for such types. +}; // can_be_passed_in_reg + +template +struct optimal_const_ref { using type = T const &; }; + +template +struct optimal_const_ref> { using type = std::basic_string_view; }; + +template +struct optimal_const_ref { using type = std::span const>; }; + +template +struct [[ clang::trivial_abi ]] pass_in_reg +{ + static auto constexpr pass_by_val{ can_be_passed_in_reg }; + + using value_type = T; + using stored_type = std::conditional_t::type>; + BOOST_FORCEINLINE + constexpr pass_in_reg( auto const &... args ) noexcept requires requires { stored_type{ args... }; } : value{ args... } {} + constexpr pass_in_reg( pass_in_reg const & ) noexcept = default; + constexpr pass_in_reg( pass_in_reg && ) noexcept = default; + + stored_type value; + + [[ gnu::pure ]] BOOST_FORCEINLINE + constexpr operator stored_type const &() const noexcept { return value; } +}; // pass_in_reg +template +pass_in_reg( T ) -> pass_in_reg; + +template +struct [[ clang::trivial_abi ]] pass_rv_in_reg +{ + static auto constexpr pass_by_val{ can_be_passed_in_reg }; + + using value_type = T; + using stored_type = std::conditional_t; + + constexpr pass_rv_in_reg( T && u ) noexcept : value{ std::move( u ) } {} // move for not-trivially-moveable yet trivial_abi types (that can be passed in reg) + + stored_type value; + + [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type const & () const & noexcept { return value ; } + [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type &&() && noexcept { return std::move( value ); } +}; // pass_rv_in_reg + +template bool constexpr reg { can_be_passed_in_reg }; +template bool constexpr reg>{ true }; +template bool constexpr reg>{ true }; + +template +concept Reg = reg; + +// 'Explicit IPA SROA' / pass-in-reg helper end +//////////////////////////////////////////////////////////////////////////////// + +PSI_WARNING_DISABLE_POP() + +//------------------------------------------------------------------------------ +} // namespace psi::vm +//------------------------------------------------------------------------------ diff --git a/include/psi/vm/containers/b+tree.hpp b/include/psi/vm/containers/b+tree.hpp index 6cf7d58..ba830e2 100644 --- a/include/psi/vm/containers/b+tree.hpp +++ b/include/psi/vm/containers/b+tree.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -43,91 +44,12 @@ namespace detail } } // namespace detail -//////////////////////////////////////////////////////////////////////////////// -// Modern(ized) attempt at 'automatized' boost::call_traits primarily to support -// efficient transparent comparators & non-inlined generic lookup functions -// which cause neither unnecessary copies of non-trivial types nor pass-by-ref -// of trivial ones. -// Largely still WiP... -// Essentially this is 'explicit IPA SROA'. -// https://gcc.gnu.org/onlinedocs/gccint/passes-and-files-of-the-compiler/inter-procedural-optimization-passes.html -//////////////////////////////////////////////////////////////////////////////// - -template -bool constexpr can_be_passed_in_reg -{ - ( - std::is_trivial_v && - ( sizeof( T ) <= 2 * sizeof( void * ) ) // assuming a sane ABI like SysV (ignoring the MS x64 disaster) - ) -#if defined( __GNUC__ ) || defined( __clang__ ) - || // detect SIMD types (this could also produce false positives for large compiler-native vectors that do not fit into the register file) - requires{ __builtin_convertvector( T{}, T ); } -#endif - // This is certainly not an exhaustive list/'trait' - certain types that can - // be passed in reg cannot be detected as such by existing compiler - // functionality, e.g. Homogeneous Vector Aggregates - // https://devblogs.microsoft.com/cppblog/introducing-vector-calling-convention - // users are encouraged to provide specializations for such types. -}; // can_be_passed_in_reg - -template -struct optimal_const_ref { using type = T const &; }; - -template -struct optimal_const_ref> { using type = std::basic_string_view; }; - -template -struct optimal_const_ref { using type = std::span const>; }; - -template -struct [[ clang::trivial_abi ]] pass_in_reg -{ - static auto constexpr pass_by_val{ can_be_passed_in_reg }; - - using value_type = T; - using stored_type = std::conditional_t::type>; - - constexpr pass_in_reg( T const & u ) noexcept : val{ u } {} - - stored_type val; - - [[ gnu::pure ]] BOOST_FORCEINLINE - constexpr operator stored_type const &() const noexcept { return val; } -}; // pass_in_reg - -template -struct [[ clang::trivial_abi ]] pass_rv_in_reg -{ - static auto constexpr pass_by_val{ can_be_passed_in_reg }; - - using value_type = T; - using stored_type = std::conditional_t; - - constexpr pass_rv_in_reg( T && u ) noexcept : val{ std::move( u ) } {} // move for not-trivially-moveable yet trivial_abi types (that can be passed in reg) - - stored_type val; - - [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type const & () const noexcept { return val ; } - [[ gnu::pure ]] BOOST_FORCEINLINE constexpr operator stored_type &&() noexcept { return std::move( val ); } -}; // pass_rv_in_reg - template concept LookupType = transparent_comparator || std::is_same_v; template concept InsertableType = ( transparent_comparator && std::is_convertible_v ) || std::is_same_v; -template bool constexpr reg { can_be_passed_in_reg }; -template bool constexpr reg>{ true }; -template bool constexpr reg>{ true }; - -template -concept Reg = reg; - -// 'Explicit IPA SROA' / pass-in-reg helper end -//////////////////////////////////////////////////////////////////////////////// - // user specializations are allowed and intended: @@ -174,8 +96,11 @@ class bptree_base { storage_result success{ nodes_.map_file( file, policy ) }; # ifndef NDEBUG - p_hdr_ = &hdr(); - p_nodes_ = nodes_.data(); + if ( std::move( success ) ) + { + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); + } # endif if ( std::move( success ) && nodes_.empty() ) hdr() = {}; diff --git a/src/containers/b+tree.cpp b/src/containers/b+tree.cpp index efe91eb..7a60074 100644 --- a/src/containers/b+tree.cpp +++ b/src/containers/b+tree.cpp @@ -11,19 +11,26 @@ namespace psi::vm // https://www.programiz.com/dsa/b-plus-tree // https://courses.cs.washington.edu/courses/cse332/23su/lectures/9_B_Trees.pdf (version, as this impl, which has different key counts in inner vs leaf nodes) // https://www.geeksforgeeks.org/b-trees-implementation-in-c -// https://github.com/jeffplaisance/BppTree // https://flatcap.github.io/linux-ntfs/ntfs/concepts/tree/index.html // https://benjamincongdon.me/blog/2021/08/17/B-Trees-More-Than-I-Thought-Id-Want-to-Know // https://stackoverflow.com/questions/59362113/b-tree-minimum-internal-children-count-explanation // https://web.archive.org/web/20190126073810/http://supertech.csail.mit.edu/cacheObliviousBTree.html // https://www.researchgate.net/publication/220225482_Cache-Oblivious_Databases_Limitations_and_Opportunities // https://www.postgresql.org/docs/current/btree.html -// https://abseil.io/about/design/btree // https://www.scylladb.com/2021/11/23/the-taming-of-the-b-trees // https://www.scattered-thoughts.net/writing/smolderingly-fast-btrees +// B+tree vs LSM-tree https://www.usenix.org/conference/fast22/presentation/qiao // Data Structure Visualizations https://www.cs.usfca.edu/~galles/visualization/Algorithms.html // Griffin: Fast Transactional Database Index with Hash and B+-Tree https://ieeexplore.ieee.org/abstract/document/10678674 // Restructuring the concurrent B+-tree with non-blocked search operations https://www.sciencedirect.com/science/article/abs/pii/S002002550200261X +// Cache-Friendly Search Trees https://arxiv.org/pdf/1907.01631 +// ZBTree: A Fast and Scalable B+-Tree for Persistent Memory https://ieeexplore.ieee.org/document/10638243 + +// https://abseil.io/about/design/btree +// https://github.com/tlx/tlx/tree/master/tlx/container +// https://github.com/jeffplaisance/BppTree +// https://github.com/postgres/postgres/blob/master/src/backend/access/nbtree/README +// https://github.com/scylladb/scylladb/blob/master/utils/intrusive_btree.hh // https://en.wikipedia.org/wiki/Judy_array @@ -50,8 +57,11 @@ bptree_base::map_memory( std::uint32_t const initial_capacity_as_number_of_nodes { storage_result success{ nodes_.map_memory( initial_capacity_as_number_of_nodes, value_init ) }; #ifndef NDEBUG - p_hdr_ = &hdr(); - p_nodes_ = nodes_.data(); + if ( std::move( success ) ) + { + p_hdr_ = &hdr(); + p_nodes_ = nodes_.data(); + } #endif if ( std::move( success ) ) {