diff --git a/.clang-format b/.clang-format index f8238280d..5a4332b47 100644 --- a/.clang-format +++ b/.clang-format @@ -45,7 +45,6 @@ ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: true -PointerAlignment: Left DisableFormat: false ExperimentalAutoDetectBinPacking: false ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index 8bfc3150d..c47953a94 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -53,25 +53,28 @@ include (TargetArch) target_architecture(COMPILER_TARGET_ARCH) # Configure compile options -string(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -string(REPLACE "-O2" "-O3" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") -string(REPLACE "-O2" "-O3" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") -if (NOT ${COMPILER_TARGET_ARCH} STREQUAL "e2k") - string(REPLACE "-g" "-g1" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -else() - string(REPLACE "-g" "-g0" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -endif() - +if (MSVC) + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g1") + set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g1") + set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") + set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG") +else () + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g1") + set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g1") + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") + set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") +endif () if (${COMPILER_TARGET_ARCH} STREQUAL "e2k") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g0") add_definitions(-D__E2K__) add_definitions(-D__LCC__) -endif() +endif () if (NOT MSVC AND NOT APPLE) check_linker_flag (-gz cxx_linker_supports_gz) if (cxx_linker_supports_gz) - set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -gz") - endif () + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -gz") + endif () endif () if (MSVC) diff --git a/cpp_src/client/coroqueryresults.cc b/cpp_src/client/coroqueryresults.cc index fdc295c75..ee3c1649a 100644 --- a/cpp_src/client/coroqueryresults.cc +++ b/cpp_src/client/coroqueryresults.cc @@ -212,9 +212,10 @@ Item CoroQueryResults::Iterator::GetItem() { if (err.ok()) { return item; } + return Item(); } catch (const Error &) { + return Item(); } - return Item(); } int64_t CoroQueryResults::Iterator::GetLSN() { diff --git a/cpp_src/client/queryresults.cc b/cpp_src/client/queryresults.cc index 83b1755c3..639d4cbab 100644 --- a/cpp_src/client/queryresults.cc +++ b/cpp_src/client/queryresults.cc @@ -250,9 +250,10 @@ Item QueryResults::Iterator::GetItem() { if (err.ok()) { return item; } + return Item(); } catch (const Error &) { + return Item(); } - return Item(); } int64_t QueryResults::Iterator::GetLSN() { diff --git a/cpp_src/client/resultserializer.cc b/cpp_src/client/resultserializer.cc index cc23e0be8..e6cb75c3e 100644 --- a/cpp_src/client/resultserializer.cc +++ b/cpp_src/client/resultserializer.cc @@ -50,6 +50,8 @@ void ResultSerializer::GetRawQueryParams(ResultSerializer::QueryParams& ret, con case QueryResultExplain: ret.explainResults = std::string(data); break; + default: + throw Error(errLogic, "Unexpected Query tag: %d", tag); } } } diff --git a/cpp_src/client/rpcclient.cc b/cpp_src/client/rpcclient.cc index f0f4b4b72..48189bf15 100644 --- a/cpp_src/client/rpcclient.cc +++ b/cpp_src/client/rpcclient.cc @@ -3,6 +3,7 @@ #include #include "client/itemimpl.h" #include "core/namespacedef.h" +#include "core/schema.h" #include "gason/gason.h" #include "tools/cpucheck.h" #include "tools/errors.h" @@ -98,6 +99,7 @@ void RPCClient::run(size_t thIdx) { workers_[thIdx].stop_.start(); delayedUpdates_.clear(); + serialDelays_ = 0; for (size_t i = thIdx; int(i) < config_.ConnPoolSize; i += config_.WorkerThreads) { connections_[i].reset(new cproto::ClientConnection(workers_[thIdx].loop_, &connectData_, @@ -717,21 +719,39 @@ void RPCClient::onUpdates(net::cproto::RPCAnswer& ans, cproto::ClientConnection* // If tagsMatcher has been updated but there is no bundled tagsMatcher in cjson // then we need to ask server to send tagsMatcher. + ++serialDelays_; // Delay this update and all the further updates until we get responce from server. ans.EnsureHold(); delayedUpdates_.emplace_back(std::move(ans)); QueryResults* qr = new QueryResults; Select(Query(std::string(nsName)).Limit(0), *qr, - InternalRdxContext(nullptr, - [=](const Error& err) { - delete qr; - // If there are delayed updates then send them to client - auto uq = std::move(delayedUpdates_); - delayedUpdates_.clear(); - if (err.ok()) - for (auto& a1 : uq) onUpdates(a1, conn); - }), + InternalRdxContext( + nullptr, + [this, qr, conn](const Error& err) { + delete qr; + // If there are delayed updates then send them to client + auto uq = std::move(delayedUpdates_); + delayedUpdates_.clear(); + + if (!err.ok() || serialDelays_ > 1) { + // This update was already dealyed, but was not able to synchronize tagsmatcher. + // Such situation usually means, that master's namespace was recreated and must be synchronized via force + // sync. + // Current fix is suboptimal and in some cases even incorrect (but still better, than previous + // implementation) - proper fix requires some versioning info about namespaces, which exists + // in v4 only + std::string_view nsName(std::string_view(uq.front().GetArgs(1)[1])); + logPrintf( + LogWarning, + "[repl:%s] Unable to sync tags matcher via online-replication (err: '%s'). Calling UpdatesLost fallback", + nsName, err.what()); + serialDelays_ = 0; + observers_.OnUpdatesLost(nsName); + } else { + for (auto& a1 : uq) onUpdates(a1, conn); + } + }), conn); return; } else { @@ -748,7 +768,17 @@ void RPCClient::onUpdates(net::cproto::RPCAnswer& ans, cproto::ClientConnection* ns->tagsMatcher_.deserialize(rdser, wrec.itemModify.tmVersion, ns->tagsMatcher_.stateToken()); } } + } else if (wrec.type == WalTagsMatcher) { + TagsMatcher tm; + Serializer ser(wrec.data.data(), wrec.data.size()); + const auto version = ser.GetVarint(); + const auto stateToken = ser.GetVarint(); + tm.deserialize(ser, version, stateToken); + auto ns = getNamespace(nsName); + std::lock_guard lck(ns->lck_); + ns->tagsMatcher_ = std::move(tm); } + serialDelays_ = 0; observers_.OnWALUpdate(LSNPair(lsn, originLSN), nsName, wrec); } diff --git a/cpp_src/client/rpcclient.h b/cpp_src/client/rpcclient.h index eefb2cf03..a906fe898 100644 --- a/cpp_src/client/rpcclient.h +++ b/cpp_src/client/rpcclient.h @@ -90,6 +90,7 @@ class RPCClient { ev::async stop_; std::atomic_bool running; }; + Error selectImpl(std::string_view query, QueryResults &result, cproto::ClientConnection *, seconds netTimeout, const InternalRdxContext &ctx); Error selectImpl(const Query &query, QueryResults &result, cproto::ClientConnection *, seconds netTimeout, @@ -122,6 +123,7 @@ class RPCClient { UpdatesObservers observers_; std::atomic updatesConn_; std::vector delayedUpdates_; + uint64_t serialDelays_ = 0; cproto::ClientConnection::ConnectData connectData_; }; diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100755 index 000000000..d189d3841 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +echo "## choose latest release rpm file" +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 3) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 4) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip; +unzip -o big.zip # unzips into mydb_big.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"replication":{"data_hash":24651210926,"data_count":3}}, +{"replication":{"data_hash":6252344969,"data_count":1}}, +{"replication":{"data_hash":37734732881,"data_count":28}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":1024095024522,"data_count":1145}}, +{"replication":{"data_hash":8373644068,"data_count":1315}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":7404222244,"data_count":97}}, +{"replication":{"data_hash":94132837196,"data_count":4}}, +{"replication":{"data_hash":1896088071,"data_count":2}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":-672103903,"data_count":33538}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":6833710705,"data_count":1}}, +{"replication":{"data_hash":5858155773472,"data_count":4500}}, +{"replication":{"data_hash":-473221280268823592,"data_count":65448}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":8288213744,"data_count":3}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":354171024786967,"data_count":3941}}, +{"replication":{"data_hash":-6520334670,"data_count":35886}}, +{"replication":{"data_hash":112772074632,"data_count":281}}, +{"replication":{"data_hash":-12679568198538,"data_count":1623116}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc index 77c3abb70..1b5853d70 100644 --- a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc @@ -191,7 +191,43 @@ Error CommandsExecutor::runImpl(const std::string& dsn, Args&&... a using reindexer::net::ev::sig; assertrx(!executorThr_.joinable()); - auto fn = [this](const std::string& dsn, Args&&... args) { + auto fnLoop = [this](const std::string& dsn, Args&&... args) { + std::string outputMode; + Error err; + if (reindexer::fs::ReadFile(reindexer::fs::JoinPath(reindexer::fs::GetHomeDir(), kConfigFile), outputMode) > 0) { + try { + gason::JsonParser jsonParser; + gason::JsonNode value = jsonParser.Parse(reindexer::giftStr(outputMode)); + for (auto node : value) { + WrSerializer ser; + reindexer::jsonValueToString(node.value, ser, 0, 0, false); + variables_[kVariableOutput] = std::string(ser.Slice()); + } + } catch (const gason::Exception& e) { + err = Error(errParseJson, "Unable to parse output mode: %s", e.what()); + } + } + if (err.ok() && variables_.empty()) { + variables_[kVariableOutput] = kOutputModeJson; + } + if (err.ok() && !uri_.parse(dsn)) { + err = Error(errNotValid, "Cannot connect to DB: Not a valid uri"); + } + if (err.ok()) err = db().Connect(dsn, std::forward(args)...); + if (err.ok()) { + loop_.spawn( + [this] { + // This coroutine should prevent loop from stopping for core::Reindexer + stopCh_.pop(); + }, + k8KStack); + } + std::lock_guard lck(mtx_); + status_.running = err.ok(); + status_.err = std::move(err); + }; + + auto fnThread = [this, &fnLoop](const std::string& dsn, Args&&... args) { sig sint; sint.set(loop_); sint.set([this](sig&) { cancelCtx_.Cancel(); }); @@ -214,49 +250,13 @@ Error CommandsExecutor::runImpl(const std::string& dsn, Args&&... a }); }); cmdAsync_.start(); - - auto fn = [this](const std::string& dsn, Args&&... args) { - std::string outputMode; - Error err; - if (reindexer::fs::ReadFile(reindexer::fs::JoinPath(reindexer::fs::GetHomeDir(), kConfigFile), outputMode) > 0) { - try { - gason::JsonParser jsonParser; - gason::JsonNode value = jsonParser.Parse(reindexer::giftStr(outputMode)); - for (auto node : value) { - WrSerializer ser; - reindexer::jsonValueToString(node.value, ser, 0, 0, false); - variables_[kVariableOutput] = std::string(ser.Slice()); - } - } catch (const gason::Exception& e) { - err = Error(errParseJson, "Unable to parse output mode: %s", e.what()); - } - } - if (err.ok() && variables_.empty()) { - variables_[kVariableOutput] = kOutputModeJson; - } - if (err.ok() && !uri_.parse(dsn)) { - err = Error(errNotValid, "Cannot connect to DB: Not a valid uri"); - } - if (err.ok()) err = db().Connect(dsn, std::forward(args)...); - if (err.ok()) { - loop_.spawn( - [this] { - // This coroutine should prevent loop from stopping for core::Reindexer - stopCh_.pop(); - }, - k8KStack); - } - std::lock_guard lck(mtx_); - status_.running = err.ok(); - status_.err = std::move(err); - }; - loop_.spawn(std::bind(fn, std::cref(dsn), std::forward(args)...)); + loop_.spawn(std::bind(fnLoop, std::cref(dsn), std::forward(args)...)); loop_.run(); }; setStatus(Status()); - executorThr_ = std::thread(std::bind(fn, std::cref(dsn), std::forward(args)...)); + executorThr_ = std::thread(std::bind(fnThread, std::cref(dsn), std::forward(args)...)); auto status = GetStatus(); while (!status.running && status.err.ok()) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); diff --git a/cpp_src/core/cbinding/reindexer_c.cc b/cpp_src/core/cbinding/reindexer_c.cc index 134937df5..b86690f98 100644 --- a/cpp_src/core/cbinding/reindexer_c.cc +++ b/cpp_src/core/cbinding/reindexer_c.cc @@ -233,6 +233,9 @@ reindexer_ret reindexer_modify_item_packed(uintptr_t rx, reindexer_buffer args, case ModeDelete: err = rdxKeeper.db().Delete(ns, item, *res); break; + default: + err = Error(errParams, "Unexpected ItemModifyMode: %d", mode); + break; } } else { switch (mode) { @@ -248,6 +251,9 @@ reindexer_ret reindexer_modify_item_packed(uintptr_t rx, reindexer_buffer args, case ModeDelete: err = rdxKeeper.db().Delete(ns, item); break; + default: + err = Error(errParams, "Unexpected ItemModifyMode: %d", mode); + break; } if (err.ok()) { res->AddItem(item); diff --git a/cpp_src/core/cjson/jsonbuilder.cc b/cpp_src/core/cjson/jsonbuilder.cc index bb8905d7a..2c57fcb51 100644 --- a/cpp_src/core/cjson/jsonbuilder.cc +++ b/cpp_src/core/cjson/jsonbuilder.cc @@ -3,7 +3,8 @@ namespace reindexer { -JsonBuilder::JsonBuilder(WrSerializer &ser, ObjType type, const TagsMatcher *tm) : ser_(&ser), tm_(tm), type_(type) { +JsonBuilder::JsonBuilder(WrSerializer &ser, ObjType type, const TagsMatcher *tm, bool emitTrailingForFloat) + : ser_(&ser), tm_(tm), type_(type), emitTrailingForFloat_(emitTrailingForFloat) { switch (type_) { case ObjType::TypeArray: (*ser_) << '['; @@ -38,12 +39,12 @@ JsonBuilder &JsonBuilder::End() { JsonBuilder JsonBuilder::Object(std::string_view name, int /*size*/) { putName(name); - return JsonBuilder(*ser_, ObjType::TypeObject, tm_); + return JsonBuilder(*ser_, ObjType::TypeObject, tm_, emitTrailingForFloat_); } JsonBuilder JsonBuilder::Array(std::string_view name, int /*size*/) { putName(name); - return JsonBuilder(*ser_, ObjType::TypeArray, tm_); + return JsonBuilder(*ser_, ObjType::TypeArray, tm_, emitTrailingForFloat_); } void JsonBuilder::putName(std::string_view name) { diff --git a/cpp_src/core/cjson/jsonbuilder.h b/cpp_src/core/cjson/jsonbuilder.h index ccdc04b07..b46555ebe 100644 --- a/cpp_src/core/cjson/jsonbuilder.h +++ b/cpp_src/core/cjson/jsonbuilder.h @@ -10,16 +10,18 @@ namespace reindexer { class JsonBuilder { public: JsonBuilder() noexcept : ser_(nullptr), tm_(nullptr) {} - JsonBuilder(WrSerializer &ser, ObjType type = ObjType::TypeObject, const TagsMatcher *tm = nullptr); + JsonBuilder(WrSerializer &ser, ObjType type = ObjType::TypeObject, const TagsMatcher *tm = nullptr, bool emitTrailingForFloat = true); ~JsonBuilder() { End(); } JsonBuilder(const JsonBuilder &) = delete; - JsonBuilder(JsonBuilder &&other) : ser_(other.ser_), tm_(other.tm_), type_(other.type_), count_(other.count_) { + JsonBuilder(JsonBuilder &&other) noexcept + : ser_(other.ser_), tm_(other.tm_), type_(other.type_), count_(other.count_), emitTrailingForFloat_(other.emitTrailingForFloat_) { other.type_ = ObjType::TypePlain; } JsonBuilder &operator=(const JsonBuilder &) = delete; JsonBuilder &operator=(JsonBuilder &&) = delete; void SetTagsMatcher(const TagsMatcher *tm) noexcept { tm_ = tm; } + void EmitTrailingForFloat(bool val) noexcept { emitTrailingForFloat_ = val; } /// Start new object JsonBuilder Object(std::string_view name = {}, int size = KUnknownFieldSize); @@ -56,12 +58,22 @@ class JsonBuilder { JsonBuilder &Put(std::string_view name, Uuid arg, int offset = 0); JsonBuilder &Put(std::nullptr_t, std::string_view arg, int offset = 0) { return Put(std::string_view{}, arg, offset); } JsonBuilder &Put(std::string_view name, const char *arg, int offset = 0) { return Put(name, std::string_view(arg), offset); } - template ::value || std::is_floating_point::value>::type * = nullptr> + template ::value>::type * = nullptr> JsonBuilder &Put(std::string_view name, const T &arg, int /*offset*/ = 0) { putName(name); (*ser_) << arg; return *this; } + template ::value>::type * = nullptr> + JsonBuilder &Put(std::string_view name, const T &arg, int /*offset*/ = 0) { + putName(name); + if (emitTrailingForFloat_) { + (*ser_) << arg; + } else { + ser_->PutDoubleStrNoTrailing(arg); + } + return *this; + } template JsonBuilder &Put(int tagName, const T &arg, int offset = 0) { return Put(getNameByTag(tagName), arg, offset); @@ -86,6 +98,7 @@ class JsonBuilder { const TagsMatcher *tm_; ObjType type_ = ObjType::TypePlain; int count_ = 0; + bool emitTrailingForFloat_ = true; }; } // namespace reindexer diff --git a/cpp_src/core/cjson/tagsmatcherimpl.h b/cpp_src/core/cjson/tagsmatcherimpl.h index 80577249c..c310ade55 100644 --- a/cpp_src/core/cjson/tagsmatcherimpl.h +++ b/cpp_src/core/cjson/tagsmatcherimpl.h @@ -186,7 +186,6 @@ class TagsMatcherImpl { names2tags_.emplace(name, tag); tags2names_[tag] = name; } - version_++; // assert(ser.Eof()); } void deserialize(Serializer &ser, int version, int stateToken) { diff --git a/cpp_src/core/dbconfig.cc b/cpp_src/core/dbconfig.cc index 60ebb6550..0a7635463 100644 --- a/cpp_src/core/dbconfig.cc +++ b/cpp_src/core/dbconfig.cc @@ -267,10 +267,10 @@ void ReplicationConfigData::GetYAML(WrSerializer &ser) const { "# Force resync on wrong data hash conditions\n" "force_sync_on_wrong_data_hash: " + (forceSyncOnWrongDataHash ? "true" : "false") + "\n" "\n" - "# Resync timeout on network errors" + "# Resync timeout on network errors\n" "retry_sync_interval_sec: " + std::to_string(retrySyncIntervalSec) + "\n" "\n" - "# Count of online replication erros, which will be merged in single error message" + "# Count of online replication erros, which will be merged in single error message\n" "online_repl_errors_threshold: " + std::to_string(onlineReplErrorsThreshold) + "\n" "\n" "# List of namespaces for replication. If emply, all namespaces\n" diff --git a/cpp_src/core/defnsconfigs.h b/cpp_src/core/defnsconfigs.h index 04fec8aa3..1601c2b68 100644 --- a/cpp_src/core/defnsconfigs.h +++ b/cpp_src/core/defnsconfigs.h @@ -75,6 +75,7 @@ const std::vector kDefDBConfig = { "replication":{ "role":"none", "master_dsn":"cproto://127.0.0.1:6534/db", + "server_id":0, "cluster_id":2, "force_sync_on_logic_error": false, "force_sync_on_wrong_data_hash": false, diff --git a/cpp_src/core/expressiontree.h b/cpp_src/core/expressiontree.h index 0eeaced61..6afcede41 100644 --- a/cpp_src/core/expressiontree.h +++ b/cpp_src/core/expressiontree.h @@ -37,18 +37,6 @@ struct Skip {}; /// For detailed documentation see expressiontree.md template class ExpressionTree { - template - class Ref { - public: - explicit Ref(T& v) noexcept : ptr_{&v} {} - operator T&() noexcept { return *ptr_; } - operator const T&() const noexcept { return *ptr_; } - bool operator==(const Ref& other) const noexcept(noexcept(std::declval() == std::declval())) { return *ptr_ == *other.ptr_; } - - private: - T* ptr_; - }; - template struct OverloadResolutionHelper : private OverloadResolutionHelper { constexpr static T resolve(std::function) noexcept; @@ -77,11 +65,6 @@ class ExpressionTree { R operator()(Arg&& arg) const noexcept(noexcept(std::declval()(std::forward(arg)))) { return functor_(std::forward(arg)); } - R operator()(const Ref& arg) const noexcept(noexcept(std::declval()(arg))) { return functor_(arg); } - R operator()(Ref& arg) const noexcept(noexcept(std::declval()(arg))) { return functor_(arg); } - R operator()(Ref&& arg) const noexcept(noexcept(std::declval()(std::forward>(arg)))) { - return functor_(std::forward>(arg)); - } private: F functor_; @@ -102,18 +85,6 @@ class ExpressionTree { R operator()(Arg&& arg) const noexcept(noexcept(std::declval()(std::forward(arg)))) { return functor_(std::forward(arg)); } - template - R operator()(const Ref& arg) const noexcept(noexcept(std::declval()(arg))) { - return functor_(arg); - } - template - R operator()(Ref& arg) const noexcept(noexcept(std::declval()(arg))) { - return functor_(arg); - } - template - R operator()(Ref&& arg) const noexcept(noexcept(std::declval()(std::forward>(arg)))) { - return functor_(std::forward>(arg)); - } private: F functor_; @@ -142,9 +113,6 @@ class ExpressionTree { void operator()(const T&) const noexcept {} void operator()(T&) const noexcept {} void operator()(T&&) const noexcept {} - void operator()(const Ref&) const noexcept {} - void operator()(Ref&) const noexcept {} - void operator()(Ref&&) const noexcept {} }; template class SkipHelper { @@ -152,9 +120,6 @@ class ExpressionTree { void operator()(const T&) const noexcept {} void operator()(T&) const noexcept {} void operator()(T&&) const noexcept {} - void operator()(const Ref&) const noexcept {} - void operator()(Ref&) const noexcept {} - void operator()(Ref&&) const noexcept {} }; template class VisitorHelper, Fs...> : private SkipHelper, private VisitorHelper { @@ -190,7 +155,7 @@ class ExpressionTree { class Node { friend ExpressionTree; - using Storage = std::variant...>; + using Storage = std::variant; struct SizeVisitor { template @@ -203,7 +168,6 @@ class ExpressionTree { template struct GetVisitor { T& operator()(T& v) const noexcept { return v; } - T& operator()(Ref& r) const noexcept { return r; } template T& operator()(U&) const noexcept { assertrx(0); @@ -213,7 +177,6 @@ class ExpressionTree { template struct GetVisitor { const T& operator()(const T& v) const noexcept { return v; } - const T& operator()(const Ref& r) const noexcept { return r; } template const T& operator()(const U&) const noexcept { assertrx(0); @@ -225,62 +188,25 @@ class ExpressionTree { bool operator()(const T& lhs, const T& rhs) const noexcept(noexcept(lhs == rhs)) { return lhs == rhs; } - template - bool operator()(const Ref& lhs, const T& rhs) const noexcept(noexcept(rhs == rhs)) { - return static_cast(lhs) == rhs; - } - template - bool operator()(const T& lhs, const Ref& rhs) const noexcept(noexcept(lhs == lhs)) { - return lhs == static_cast(rhs); - } template bool operator()(const T&, const U&) const noexcept { return false; } }; - struct LazyCopyVisitor { - Storage operator()(SubTree& st) const noexcept { return st; } - template - Storage operator()(Ref& r) const { - return r; - } - template - Storage operator()(T& v) const { - return Ref{v}; - } - }; - struct DeepCopyVisitor { + struct CopyVisitor { Storage operator()(const SubTree& st) const noexcept { return st; } template - Storage operator()(const Ref& r) const { - return static_cast(r); - } - template Storage operator()(const T& v) const { return v; } }; - struct DeepRValueCopyVisitor { + struct MoveVisitor { Storage operator()(SubTree&& st) const noexcept { return std::move(st); } template - Storage operator()(Ref&& r) const { - return static_cast(r); - } - template Storage operator()(T&& v) const { return std::forward(v); } }; - struct IsRefVisitor { - template - bool operator()(const T&) const noexcept { - return false; - } - template - bool operator()(const Ref&) const noexcept { - return true; - } - }; public: Node() : storage_{std::in_place_type, 1} {} @@ -325,13 +251,9 @@ class ExpressionTree { bool IsSubTree() const noexcept { return storage_.index() == 0; } bool IsLeaf() const noexcept { return !IsSubTree(); } template - bool Holds() const noexcept { + bool Is() const noexcept { return std::holds_alternative(storage_); } - template - bool HoldsOrReferTo() const noexcept { - return std::holds_alternative(storage_) || std::holds_alternative>(storage_); - } void Append() { std::get(storage_).Append(); } void Erase(size_t length) { std::get(storage_).Erase(length); } /// Execute appropriate functor depending on content type @@ -344,22 +266,14 @@ class ExpressionTree { return std::visit(Visitor{std::forward(funcs)...}, storage_); } - Node MakeLazyCopy() & { - static const LazyCopyVisitor visitor; - return {operation, std::visit(visitor, storage_)}; - } - Node MakeDeepCopy() const& { - static const DeepCopyVisitor visitor; + Node Copy() const& { + static const CopyVisitor visitor; return {operation, std::visit(visitor, storage_)}; } - Node MakeDeepCopy() && { - static const DeepRValueCopyVisitor visitor; + Node Move() && { + static const MoveVisitor visitor; return {operation, std::visit(visitor, std::move(storage_))}; } - bool IsRef() const { - static const IsRefVisitor visitor; - return std::visit(visitor, storage_); - } template void SetValue(T&& v) { storage_ = std::forward(v); @@ -385,13 +299,13 @@ class ExpressionTree { ExpressionTree& operator=(ExpressionTree&&) = default; ExpressionTree(const ExpressionTree& other) : activeBrackets_{other.activeBrackets_} { container_.reserve(other.container_.size()); - for (const Node& n : other.container_) container_.emplace_back(n.MakeDeepCopy()); + for (const Node& n : other.container_) container_.emplace_back(n.Copy()); } ExpressionTree& operator=(const ExpressionTree& other) { if (this == &other) return *this; container_.clear(); container_.reserve(other.container_.size()); - for (const Node& n : other.container_) container_.emplace_back(n.MakeDeepCopy()); + for (const Node& n : other.container_) container_.emplace_back(n.Copy()); activeBrackets_ = other.activeBrackets_; return *this; } @@ -463,11 +377,6 @@ class ExpressionTree { container_.reserve(container_.size() + (end.PlainIterator() - begin.PlainIterator())); append(begin, end); } - class iterator; - void LazyAppend(iterator begin, iterator end) { - container_.reserve(container_.size() + (end.PlainIterator() - begin.PlainIterator())); - lazyAppend(begin, end); - } /// Appends value as first child of the root template @@ -534,9 +443,9 @@ class ExpressionTree { return i + Size(i); } template - bool HoldsOrReferTo(size_t i) const noexcept { + bool Is(size_t i) const noexcept { assertrx(i < Size()); - return container_[i].template HoldsOrReferTo(); + return container_[i].template Is(); } bool IsSubTree(size_t i) const noexcept { assertrx(i < Size()); @@ -729,28 +638,6 @@ class ExpressionTree { [this, op](const auto& v) -> void { this->Append(op, v); }); } } - - void lazyAppend(iterator begin, iterator end) { - for (; begin != end; ++begin) { - const OpType op = begin->operation; - begin->template InvokeAppropriate( - [this, &begin, op](const SubTree& b) { - OpenBracket(op); - std::get(container_.back().storage_).CopyPayloadFrom(b); - lazyAppend(begin.begin(), begin.end()); - CloseBracket(); - }, - [this, op](auto& v) -> void { this->Append(op, Ref>{v}); }); - } - } - - ExpressionTree makeLazyCopy() & { - ExpressionTree result; - result.container_.reserve(container_.size()); - for (Node& n : container_) result.container_.emplace_back(n.MakeLazyCopy()); - result.activeBrackets_ = activeBrackets_; - return result; - } }; } // namespace reindexer diff --git a/cpp_src/core/ft/areaholder.h b/cpp_src/core/ft/areaholder.h index e4db54a9f..fb186f6b0 100644 --- a/cpp_src/core/ft/areaholder.h +++ b/cpp_src/core/ft/areaholder.h @@ -39,7 +39,8 @@ class AreaBuffer { [[nodiscard]] bool Empty() const noexcept { return data_.empty(); } void Commit() { if (!data_.empty()) { - boost::sort::pdqsort(data_.begin(), data_.end(), [](const Area &rhs, const Area &lhs) { return rhs.start < lhs.start; }); + boost::sort::pdqsort_branchless(data_.begin(), data_.end(), + [](const Area &rhs, const Area &lhs) noexcept { return rhs.start < lhs.start; }); for (auto vit = data_.begin() + 1; vit != data_.end(); ++vit) { auto prev = vit - 1; if (vit->Concat(*prev)) { diff --git a/cpp_src/core/ft/ft_fast/dataprocessor.cc b/cpp_src/core/ft/ft_fast/dataprocessor.cc index 772b1c785..1b3df8eb0 100644 --- a/cpp_src/core/ft/ft_fast/dataprocessor.cc +++ b/cpp_src/core/ft/ft_fast/dataprocessor.cc @@ -149,13 +149,16 @@ size_t DataProcessor::buildWordsMap(words_map &words_um) { auto &vdocs = holder_.vdocs_; const int fieldscount = fieldSize_; size_t offset = holder_.vdocsOffset_; + auto cycleSize = vdocsTexts.size() / maxIndexWorkers + (vdocsTexts.size() % maxIndexWorkers ? 1 : 0); // build words map parallel in maxIndexWorkers threads - auto worker = [this, &ctxs, &vdocsTexts, offset, maxIndexWorkers, fieldscount, &cfg, &vdocs](int i) { + auto worker = [this, &ctxs, &vdocsTexts, offset, cycleSize, fieldscount, &cfg, &vdocs](int i) { auto ctx = &ctxs[i]; std::string word, str; std::vector wrds; std::vector virtualWords; - for (VDocIdType j = i, sz = VDocIdType(vdocsTexts.size()); j < sz; j += maxIndexWorkers) { + size_t start = cycleSize * i; + size_t fin = std::min(cycleSize * (i + 1), vdocsTexts.size()); + for (VDocIdType j = start; j < fin; ++j) { const size_t vdocId = offset + j; auto &vdoc = vdocs[vdocId]; vdoc.wordsCount.insert(vdoc.wordsCount.begin(), fieldscount, 0.0); diff --git a/cpp_src/core/ft/ft_fast/selecter.cc b/cpp_src/core/ft/ft_fast/selecter.cc index b460ca929..40082ae17 100644 --- a/cpp_src/core/ft/ft_fast/selecter.cc +++ b/cpp_src/core/ft/ft_fast/selecter.cc @@ -335,8 +335,8 @@ void Selecter::processLowRelVariants(FtSelectContext& ctx, const FtMerge return false; }); } else { - boost::sort::pdqsort(ctx.lowRelVariants.begin(), ctx.lowRelVariants.end(), - [](FtBoundVariantEntry& l, FtBoundVariantEntry& r) noexcept { return l.proc > r.proc; }); + boost::sort::pdqsort_branchless(ctx.lowRelVariants.begin(), ctx.lowRelVariants.end(), + [](FtBoundVariantEntry& l, FtBoundVariantEntry& r) noexcept { return l.proc > r.proc; }); } auto lastVariantLen = ctx.lowRelVariants.size() ? ctx.lowRelVariants[0].GetLenCached() : -1; @@ -790,7 +790,7 @@ std::pair Selecter::calcTermRank(const TextSearchResults& r if (!termRank) return std::make_pair(termRank, field); if (holder_.cfg_->summationRanksByFieldsRatio > 0) { - std::sort(ranksInFields.begin(), ranksInFields.end()); + boost::sort::pdqsort_branchless(ranksInFields.begin(), ranksInFields.end()); double k = holder_.cfg_->summationRanksByFieldsRatio; for (auto rank : ranksInFields) { termRank += (k * rank); @@ -921,9 +921,10 @@ void Selecter::mergeIterationGroup(TextSearchResults& rawRes, index_t ra mergedPosInfo.rank = 0; } else { auto& posTmp = mergedPosInfo.posTmp; - boost::sort::pdqsort( - posTmp.begin(), posTmp.end(), - [](const std::pair& l, const std::pair& r) { return l.first < r.first; }); + boost::sort::pdqsort_branchless(posTmp.begin(), posTmp.end(), + [](const std::pair& l, + const std::pair& r) noexcept { return l.first < r.first; }); + auto last = std::unique(posTmp.begin(), posTmp.end()); posTmp.resize(last - posTmp.begin()); @@ -984,9 +985,9 @@ void Selecter::mergeResultsPart(std::vector& rawResul merged.maxRank = m.proc; } } - - boost::sort::pdqsort(merged.begin(), merged.end(), - [](const IDataHolder::MergeInfo& lhs, const IDataHolder::MergeInfo& rhs) { return lhs.proc > rhs.proc; }); + boost::sort::pdqsort_branchless( + merged.begin(), merged.end(), + [](const IDataHolder::MergeInfo& lhs, const IDataHolder::MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; }); } template @@ -1244,12 +1245,11 @@ typename IDataHolder::MergeData Selecter::mergeResults(std::vector merged_rd; std::vector idoffsets; - for (auto& rawRes : rawResults) { - boost::sort::pdqsort(rawRes.begin(), rawRes.end(), - [](const TextSearchResult& lhs, const TextSearchResult& rhs) { return lhs.proc_ > rhs.proc_; }); + boost::sort::pdqsort_branchless( + rawRes.begin(), rawRes.end(), + [](const TextSearchResult& lhs, const TextSearchResult& rhs) noexcept { return lhs.proc_ > rhs.proc_; }); } - const auto maxMergedSize = std::min(size_t(holder_.cfg_->mergeLimit), totalORVids); merged.reserve(maxMergedSize); @@ -1332,8 +1332,9 @@ typename IDataHolder::MergeData Selecter::mergeResults(std::vector rhs.proc; }); + boost::sort::pdqsort_branchless( + merged.begin(), merged.end(), + [](const IDataHolder::MergeInfo& lhs, const IDataHolder::MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; }); return merged; } diff --git a/cpp_src/core/ft/ft_fuzzy/merger/basemerger.cc b/cpp_src/core/ft/ft_fuzzy/merger/basemerger.cc index 967161001..12a57498c 100644 --- a/cpp_src/core/ft/ft_fuzzy/merger/basemerger.cc +++ b/cpp_src/core/ft/ft_fuzzy/merger/basemerger.cc @@ -93,7 +93,7 @@ SearchResult BaseMerger::Merge(MergeCtx& ctx, bool inTransaction, const reindexe data_set.AddData(it->Id(), id_ctx); } } - boost::sort::pdqsort(data_set.data_->begin(), data_set.data_->end(), [](const MergedData& lhs, const MergedData& rhs) { + boost::sort::pdqsort(data_set.data_->begin(), data_set.data_->end(), [](const MergedData& lhs, const MergedData& rhs) noexcept { if (lhs.proc_ == rhs.proc_) { return lhs.id_ < rhs.id_; } diff --git a/cpp_src/core/ft/idrelset.cc b/cpp_src/core/ft/idrelset.cc index 7f31546d1..6b87bc15c 100644 --- a/cpp_src/core/ft/idrelset.cc +++ b/cpp_src/core/ft/idrelset.cc @@ -1,4 +1,3 @@ - #include "idrelset.h" #include #include "estl/h_vector.h" @@ -84,9 +83,4 @@ int IdRelSet::Add(VDocIdType id, int pos, int field) { return back().Size(); } -void IdRelType::SimpleCommit() { - boost::sort::pdqsort(pos_.begin(), pos_.end(), - [](const IdRelType::PosType& lhs, const IdRelType::PosType& rhs) { return lhs.pos() < rhs.pos(); }); -} - } // namespace reindexer diff --git a/cpp_src/core/ft/idrelset.h b/cpp_src/core/ft/idrelset.h index 05ea5f8b7..23bc811fa 100644 --- a/cpp_src/core/ft/idrelset.h +++ b/cpp_src/core/ft/idrelset.h @@ -102,11 +102,11 @@ class IdRelType { addField(field); } void SortAndUnique() { - boost::sort::pdqsort(pos_.begin(), pos_.end()); + boost::sort::pdqsort_branchless(pos_.begin(), pos_.end()); auto last = std::unique(pos_.begin(), pos_.end()); pos_.resize(last - pos_.begin()); } - void Clear() { + void Clear() noexcept { usedFieldsMask_ = 0; #ifdef REINDEXER_FT_EXTRA_DEBUG pos_.clear(); @@ -116,7 +116,11 @@ class IdRelType { } size_t Size() const noexcept { return pos_.size(); } size_t size() const noexcept { return pos_.size(); } - void SimpleCommit(); + void SimpleCommit() noexcept { + boost::sort::pdqsort_branchless( + pos_.begin(), pos_.end(), + [](const IdRelType::PosType& lhs, const IdRelType::PosType& rhs) noexcept { return lhs.pos() < rhs.pos(); }); + } const RVector& Pos() const noexcept { return pos_; } uint64_t UsedFieldsMask() const noexcept { return usedFieldsMask_; } size_t HeapSize() const noexcept { return heapSize(pos_); } @@ -141,7 +145,7 @@ class IdRelType { class IdRelSet : public std::vector { public: int Add(VDocIdType id, int pos, int field); - void SimpleCommit() { + void SimpleCommit() noexcept { for (auto& val : *this) val.SimpleCommit(); } diff --git a/cpp_src/core/ft/numtotext.cc b/cpp_src/core/ft/numtotext.cc index fc91c1848..44fe81df2 100644 --- a/cpp_src/core/ft/numtotext.cc +++ b/cpp_src/core/ft/numtotext.cc @@ -45,8 +45,9 @@ static std::string_view getNumorder(int numorder, int i) { return sextillion[i]; case Septillion: return septillion[i]; + default: + throw Error(errParams, "Incorrect order [%s]: too big", numorder); } - throw Error(errParams, "Incorrect order [%s]: too big", numorder); } RX_ALWAYS_INLINE int ansiCharacterToDigit(char ch) noexcept { return static_cast(ch - 48); } @@ -73,6 +74,8 @@ static std::vector getOrders(std::string_view str) { tempString += numStr[i + 1]; tempString += numStr[i]; break; + default: + throw Error(errLogic, "Unexpected lost characters number: %d", lostChars); } } orders.emplace_back(std::move(tempString)); diff --git a/cpp_src/core/idset.h b/cpp_src/core/idset.h index 81522fcef..e2e501c00 100644 --- a/cpp_src/core/idset.h +++ b/cpp_src/core/idset.h @@ -108,7 +108,7 @@ class IdSet : public IdSetPlain { return *this; } static Ptr BuildFromUnsorted(base_idset &&ids) { - boost::sort::pdqsort(ids.begin(), ids.end()); + boost::sort::pdqsort_branchless(ids.begin(), ids.end()); ids.erase(std::unique(ids.begin(), ids.end()), ids.end()); // TODO: It would be better to integrate unique into sort return make_intrusive>(std::move(ids)); } diff --git a/cpp_src/core/index/indextext/fastindextext.cc b/cpp_src/core/index/indextext/fastindextext.cc index 12cbdcae9..c56dc44b8 100644 --- a/cpp_src/core/index/indextext/fastindextext.cc +++ b/cpp_src/core/index/indextext/fastindextext.cc @@ -123,7 +123,7 @@ template IndexMemStat FastIndexText::GetMemStat(const RdxContext &ctx) { auto ret = IndexUnordered::GetMemStat(ctx); - contexted_shared_lock lck(this->mtx_, &ctx); + contexted_shared_lock lck(this->mtx_, ctx); ret.fulltextSize = this->holder_->GetMemStat(); ret.idsetCache = this->cache_ft_ ? this->cache_ft_->GetMemStat() : LRUCacheMemStat(); return ret; diff --git a/cpp_src/core/index/keyentry.h b/cpp_src/core/index/keyentry.h index 079f543c6..9c500dba1 100644 --- a/cpp_src/core/index/keyentry.h +++ b/cpp_src/core/index/keyentry.h @@ -33,12 +33,14 @@ class KeyEntry { auto idsAsc = Sorted(ctx.getCurSortId()); size_t idx = 0; + const auto& ids2Sorts = ctx.ids2Sorts(); + [[maybe_unused]] const IdType maxRowId = IdType(ids2Sorts.size()); // For all ids of current key for (auto rowid : ids_) { - assertf(rowid < int(ctx.ids2Sorts().size()), "id=%d,ctx.ids2Sorts().size()=%d", rowid, ctx.ids2Sorts().size()); - idsAsc[idx++] = ctx.ids2Sorts()[rowid]; + assertf(rowid < maxRowId, "id=%d,ctx.ids2Sorts().size()=%d", rowid, maxRowId); + idsAsc[idx++] = ids2Sorts[rowid]; } - boost::sort::pdqsort(idsAsc.begin(), idsAsc.end()); + boost::sort::pdqsort_branchless(idsAsc.begin(), idsAsc.end()); } void Dump(std::ostream& os, std::string_view step, std::string_view offset) const { std::string newOffset; diff --git a/cpp_src/core/index/rtree/greenesplitter.h b/cpp_src/core/index/rtree/greenesplitter.h index ba707e781..c8c6a7179 100644 --- a/cpp_src/core/index/rtree/greenesplitter.h +++ b/cpp_src/core/index/rtree/greenesplitter.h @@ -1,6 +1,7 @@ #pragma once #include "splitter.h" +#include "vendor/sort/pdqsort.hpp" namespace reindexer { @@ -53,7 +54,7 @@ class GreeneSplitter : private Splitter secondSeedBoundRect.Left() + secondSeedBoundRect.Right()) { std::swap(seeds.first, seeds.second); } - std::sort(std::begin(indexes), std::end(indexes), [&src, this](size_t lhs, size_t rhs) { + boost::sort::pdqsort(std::begin(indexes), std::end(indexes), [&src, this](size_t lhs, size_t rhs) { return Base::getBoundRect(lhs < MaxEntries ? src[lhs] : this->appendingEntry_).Left() < Base::getBoundRect(rhs < MaxEntries ? src[rhs] : this->appendingEntry_).Left(); }); @@ -61,7 +62,7 @@ class GreeneSplitter : private Splitter secondSeedBoundRect.Bottom() + secondSeedBoundRect.Top()) { std::swap(seeds.first, seeds.second); } - std::sort(std::begin(indexes), std::end(indexes), [&src, this](size_t lhs, size_t rhs) { + boost::sort::pdqsort(std::begin(indexes), std::end(indexes), [&src, this](size_t lhs, size_t rhs) { return Base::getBoundRect(lhs < MaxEntries ? src[lhs] : this->appendingEntry_).Bottom() < Base::getBoundRect(rhs < MaxEntries ? src[rhs] : this->appendingEntry_).Bottom(); }); diff --git a/cpp_src/core/index/rtree/rstarsplitter.h b/cpp_src/core/index/rtree/rstarsplitter.h index a44c9748e..427a4add3 100644 --- a/cpp_src/core/index/rtree/rstarsplitter.h +++ b/cpp_src/core/index/rtree/rstarsplitter.h @@ -1,6 +1,7 @@ #pragma once #include "splitter.h" +#include "vendor/sort/pdqsort.hpp" namespace reindexer { @@ -19,12 +20,12 @@ class RStarSplitter : private SplitterFields()}; @@ -467,7 +467,6 @@ void ItemModifier::modifyField(IdType itemId, FieldData &field, Payload &pl, Var tupleValue = tupleIdx->Upsert(item.GetField(0), itemId, needClearCache); if (needClearCache && tupleIdx->IsOrdered()) indexesCacheCleaner.Add(tupleIdx->SortId()); pl.Set(0, std::move(tupleValue)); - ns_.tagsMatcher_.try_merge(item.tagsMatcher()); if (exception) { std::rethrow_exception(exception); } diff --git a/cpp_src/core/key_value_type.h b/cpp_src/core/key_value_type.h index 31c58a873..93fe19d29 100644 --- a/cpp_src/core/key_value_type.h +++ b/cpp_src/core/key_value_type.h @@ -144,8 +144,9 @@ class KeyValueType { case TAG_ARRAY: case TAG_OBJECT: case TAG_END: - throw Error(errParams, "Invalid tag type value for KeyValueType: " + std::string{TagTypeToStr(t)}); + break; } + throw Error(errParams, "Invalid tag type value for KeyValueType: " + std::string{TagTypeToStr(t)}); } template diff --git a/cpp_src/core/lsn.cc b/cpp_src/core/lsn.cc index dc88fb059..73c0130e2 100644 --- a/cpp_src/core/lsn.cc +++ b/cpp_src/core/lsn.cc @@ -3,9 +3,11 @@ namespace reindexer { -void lsn_t::GetJSON(JsonBuilder &builder) const { +void lsn_t::GetJSON(JsonBuilder& builder) const { builder.Put("server_id", Server()); builder.Put("counter", Counter()); } -} // namespace reindexer +[[noreturn]] void lsn_t::throwValidation(ErrorCode code, const char* fmt, int64_t value) { throw Error(code, fmt, value); } + +} // namespace reindexer \ No newline at end of file diff --git a/cpp_src/core/lsn.h b/cpp_src/core/lsn.h index eb1f83d15..174540da4 100644 --- a/cpp_src/core/lsn.h +++ b/cpp_src/core/lsn.h @@ -13,58 +13,53 @@ class JsonBuilder; // SSS NNN NNN NNN NNN NNN (18 decimal digits) struct lsn_t { - static const int64_t digitCountLSNMult = 1000000000000000ll; +private: + static constexpr int16_t kMinServerIDValue = 0; + static constexpr int16_t kMaxServerIDValue = 999; - static const int64_t kCounterbitCount = 48; - static const int64_t kCounterMask = (1ull << kCounterbitCount) - 1ull; + static constexpr int64_t kMaxCounter = 1000000000000000ll; + static constexpr int64_t kDefaultCounter = kMaxCounter - 1; +public: void GetJSON(JsonBuilder &builder) const; void FromJSON(const gason::JsonNode &root) { - int server = root["server_id"].As(0); - int64_t counter = root["counter"].As(digitCountLSNMult - 1ll); + const int server = root["server_id"].As(0); + const int64_t counter = root["counter"].As(kDefaultCounter); payload_ = int64_t(lsn_t(counter, server)); } - lsn_t() {} - explicit lsn_t(int64_t v) { - if ((v & kCounterMask) == kCounterMask) // init -1 - payload_ = digitCountLSNMult - 1ll; - else { - payload_ = v; - } - } - lsn_t(int64_t counter, uint8_t server) { - if ((counter & kCounterMask) == kCounterMask) counter = digitCountLSNMult - 1ll; - int64_t s = server * digitCountLSNMult; - payload_ = s + counter; + lsn_t() noexcept = default; + lsn_t(const lsn_t &) noexcept = default; + lsn_t(lsn_t &&) noexcept = default; + lsn_t &operator=(const lsn_t &) noexcept = default; + lsn_t &operator=(lsn_t &&) noexcept = default; + explicit lsn_t(int64_t v) : lsn_t(v % kMaxCounter, v / kMaxCounter) {} + lsn_t(int64_t counter, int16_t server) { + validateCounter(counter); + validateServerId(server); + payload_ = server * kMaxCounter + counter; } explicit operator int64_t() const { return payload_; } - bool operator==(lsn_t o) { return payload_ == o.payload_; } - bool operator!=(lsn_t o) { return payload_ != o.payload_; } + bool operator==(lsn_t o) const noexcept { return payload_ == o.payload_; } + bool operator!=(lsn_t o) const noexcept { return payload_ != o.payload_; } - int64_t SetServer(short s) { - if (s > 999) throw Error(errLogic, "Server id > 999"); - int64_t server = s * digitCountLSNMult; - int64_t serverOld = payload_ / digitCountLSNMult; - payload_ = payload_ - serverOld * digitCountLSNMult + server; + int64_t SetServer(short server) { + validateServerId(server); + payload_ = server * kMaxCounter + Counter(); return payload_; } - int64_t SetCounter(int64_t c) { - if (c >= digitCountLSNMult) throw Error(errLogic, "LSN Counter > digitCountLSNMult"); - int64_t server = payload_ / digitCountLSNMult; - payload_ = server * digitCountLSNMult + c; + int64_t SetCounter(int64_t counter) { + validateCounter(counter); + payload_ = Server() * kMaxCounter + counter; return payload_; } - int64_t Counter() const { - int64_t server = payload_ / digitCountLSNMult; - return payload_ - server * digitCountLSNMult; - } - short Server() const { return payload_ / digitCountLSNMult; } - bool isEmpty() const { return Counter() == digitCountLSNMult - 1ll; } + int64_t Counter() const noexcept { return payload_ % kMaxCounter; } + int16_t Server() const noexcept { return payload_ / kMaxCounter; } + bool isEmpty() const noexcept { return Counter() == kDefaultCounter; } - int compare(lsn_t o) { + int compare(lsn_t o) const { if (Server() != o.Server()) throw Error(errLogic, "Compare lsn from different server"); if (Counter() < o.Counter()) return -1; @@ -73,13 +68,28 @@ struct lsn_t { return 0; } - bool operator<(lsn_t o) { return compare(o) == -1; } - bool operator<=(lsn_t o) { return compare(o) <= 0; } - bool operator>(lsn_t o) { return compare(o) == 1; } - bool operator>=(lsn_t o) { return compare(o) >= 0; } + bool operator<(lsn_t o) const { return compare(o) == -1; } + bool operator<=(lsn_t o) const { return compare(o) <= 0; } + bool operator>(lsn_t o) const { return compare(o) == 1; } + bool operator>=(lsn_t o) const { return compare(o) >= 0; } + +private: + int64_t payload_ = kDefaultCounter; + static void validateServerId(int16_t server) { + if (server < kMinServerIDValue) { + throwValidation(errLogic, "Server id < %d", kMinServerIDValue); + } + if (server > kMaxServerIDValue) { + throwValidation(errLogic, "Server id > %d", kMaxServerIDValue); + } + } + static void validateCounter(int64_t counter) { + if (counter > kDefaultCounter) { + throwValidation(errLogic, "LSN Counter > Default LSN (%d)", kMaxCounter); + } + } -protected: - int64_t payload_ = digitCountLSNMult - 1ll; + [[noreturn]] static void throwValidation(ErrorCode, const char *, int64_t); }; struct LSNPair { diff --git a/cpp_src/core/namespace/namespace.cc b/cpp_src/core/namespace/namespace.cc index 733bb309e..01632f4f8 100644 --- a/cpp_src/core/namespace/namespace.cc +++ b/cpp_src/core/namespace/namespace.cc @@ -1,14 +1,11 @@ #include "namespace.h" #include "core/querystat.h" -#include "core/storage/storagefactory.h" #include "tools/flagguard.h" #include "tools/fsops.h" #include "tools/logger.h" namespace reindexer { -#define handleInvalidation(Fn) nsFuncWrapper - void Namespace::CommitTransaction(Transaction& tx, QueryResults& result, const RdxContext& ctx) { auto nsl = atomicLoadMainNs(); bool enablePerfCounters = nsl->enablePerfCounters_.load(std::memory_order_relaxed); @@ -23,7 +20,7 @@ void Namespace::CommitTransaction(Transaction& tx, QueryResults& result, const R if (needNamespaceCopy(nsl, tx)) { PerfStatCalculatorMT calc(nsl->updatePerfCounter_, enablePerfCounters); - auto lck = statCalculator.CreateLock(clonerMtx_, &ctx); + auto lck = statCalculator.CreateLock(clonerMtx_, ctx); nsl = ns_; if (needNamespaceCopy(nsl, tx)) { @@ -76,11 +73,11 @@ void Namespace::CommitTransaction(Transaction& tx, QueryResults& result, const R return; } } - handleInvalidation(NamespaceImpl::CommitTransaction)(tx, result, NsContext(ctx), statCalculator); + nsFuncWrapper<&NamespaceImpl::CommitTransaction>(tx, result, NsContext(ctx), statCalculator); } NamespacePerfStat Namespace::GetPerfStat(const RdxContext& ctx) { - NamespacePerfStat stats = handleInvalidation(NamespaceImpl::GetPerfStat)(ctx); + NamespacePerfStat stats = nsFuncWrapper<&NamespaceImpl::GetPerfStat>(ctx); stats.transactions = txStatsCounter_.Get(); auto copyStats = copyStatsCounter_.Get(); stats.transactions.totalCopyCount = copyStats.totalHitCount; @@ -107,7 +104,7 @@ bool Namespace::needNamespaceCopy(const NamespaceImpl::Ptr& ns, const Transactio void Namespace::doRename(const Namespace::Ptr& dst, const std::string& newName, const std::string& storagePath, const RdxContext& ctx) { std::string dbpath; const auto flushOpts = StorageFlushOpts().WithImmediateReopen(); - auto lck = handleInvalidation(NamespaceImpl::wLock)(ctx); + auto lck = nsFuncWrapper<&NamespaceImpl::wLock>(ctx); auto srcNsPtr = atomicLoadMainNs(); auto& srcNs = *srcNsPtr; srcNs.storage_.Flush(flushOpts); // Repeat flush, to raise any disk errors before attempt to close storage diff --git a/cpp_src/core/namespace/namespace.h b/cpp_src/core/namespace/namespace.h index 55dc59548..012f0856e 100644 --- a/cpp_src/core/namespace/namespace.h +++ b/cpp_src/core/namespace/namespace.h @@ -5,13 +5,82 @@ #include "bgnamespacedeleter.h" #include "core/querystat.h" #include "core/txstats.h" -#include "estl/shared_mutex.h" namespace reindexer { -#define handleInvalidation(Fn) nsFuncWrapper - class Namespace { + template + auto nsFuncWrapper(Args &&...args) const { + while (true) { + try { + auto ns = atomicLoadMainNs(); + return (*ns.*fn)(std::forward(args)...); + } catch (const Error &e) { + if (e.code() != errNamespaceInvalidated) { + throw; + } else { + std::this_thread::yield(); + } + } + } + } + + template + void nsFuncWrapper(Item &item, QueryResults &qr, NsContext ctx) const { + nsFuncWrapper(item, qr, ctx); + } + + template + void nsFuncWrapper(const Query &query, QueryResults &qr, NsContext ctx) const { + nsFuncWrapper(query, qr, ctx); + } + + template + void nsFuncWrapper(T &v, QueryResults &qr, NsContext ctx) const { + while (true) { + NamespaceImpl::Ptr ns; + bool added = false; + try { + ns = atomicLoadMainNs(); + PerfStatCalculatorMT calc(ns->updatePerfCounter_, ns->enablePerfCounters_); + CounterGuardAIR32 cg(ns->cancelCommitCnt_); + if constexpr (std::is_same_v) { + auto locker{ns->wLock(ctx.rdxContext)}; + calc.LockHit(); + cg.Reset(); + ns->checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); + qr.AddNamespace(ns, true); + added = true; + (*ns.*fn)(v, enumVal, ctx); + qr.AddItem(v, true, false); + ns->tryForceFlush(std::move(locker)); + } else if constexpr (std::is_same_v) { + auto params = longUpdDelLoggingParams_.load(std::memory_order_relaxed); + const bool isEnabled = params.thresholdUs >= 0 && !isSystemNamespaceNameFast(v.NsName()); + auto statCalculator = QueryStatCalculator(long_actions::MakeLogger(v, std::move(params)), isEnabled); + auto locker = statCalculator.CreateLock(*ns, &NamespaceImpl::wLock, ctx.rdxContext); + calc.LockHit(); + cg.Reset(); + ns->checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); + qr.AddNamespace(ns, true); + added = true; + (*ns.*fn)(v, qr, ctx); + statCalculator.LogFlushDuration(*ns, &NamespaceImpl::tryForceFlush, std::move(locker)); + } else { + static_assert(std::is_same_v || std::is_same_v); + } + return; + } catch (const Error &e) { + if (e.code() != errNamespaceInvalidated) { + throw; + } else { + if (added) qr.RemoveNamespace(ns.get()); + std::this_thread::yield(); + } + } + } + } + public: using Ptr = shared_ptr; @@ -19,103 +88,95 @@ class Namespace { : ns_(make_intrusive(name, observers)), bgDeleter_(bgDeleter) {} void CommitTransaction(Transaction &tx, QueryResults &result, const RdxContext &ctx); - std::string GetName(const RdxContext &ctx) const { return handleInvalidation(NamespaceImpl::GetName)(ctx); } - bool IsSystem(const RdxContext &ctx) const { return handleInvalidation(NamespaceImpl::IsSystem)(ctx); } - bool IsTemporary(const RdxContext &ctx) const { return handleInvalidation(NamespaceImpl::IsTemporary)(ctx); } + std::string GetName(const RdxContext &ctx) const { return nsFuncWrapper<&NamespaceImpl::GetName>(ctx); } + bool IsSystem(const RdxContext &ctx) const { return nsFuncWrapper<&NamespaceImpl::IsSystem>(ctx); } + bool IsTemporary(const RdxContext &ctx) const { return nsFuncWrapper<&NamespaceImpl::IsTemporary>(ctx); } void EnableStorage(const std::string &path, StorageOpts opts, StorageType storageType, const RdxContext &ctx) { - handleInvalidation(NamespaceImpl::EnableStorage)(path, opts, storageType, ctx); + nsFuncWrapper<&NamespaceImpl::EnableStorage>(path, opts, storageType, ctx); } void LoadFromStorage(unsigned threadsCount, const RdxContext &ctx) { - handleInvalidation(NamespaceImpl::LoadFromStorage)(threadsCount, ctx); + nsFuncWrapper<&NamespaceImpl::LoadFromStorage>(threadsCount, ctx); } - void DeleteStorage(const RdxContext &ctx) { handleInvalidation(NamespaceImpl::DeleteStorage)(ctx); } - uint32_t GetItemsCount() { return handleInvalidation(NamespaceImpl::GetItemsCount)(); } - void AddIndex(const IndexDef &indexDef, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::AddIndex)(indexDef, ctx); } - void UpdateIndex(const IndexDef &indexDef, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::UpdateIndex)(indexDef, ctx); } - void DropIndex(const IndexDef &indexDef, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::DropIndex)(indexDef, ctx); } - void SetSchema(std::string_view schema, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::SetSchema)(schema, ctx); } - std::string GetSchema(int format, const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetSchema)(format, ctx); } - std::shared_ptr GetSchemaPtr(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetSchemaPtr)(ctx); } - void Insert(Item &item, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Insert)(item, ctx); } + void DeleteStorage(const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::DeleteStorage>(ctx); } + uint32_t GetItemsCount() { return nsFuncWrapper<&NamespaceImpl::GetItemsCount>(); } + void AddIndex(const IndexDef &indexDef, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::AddIndex>(indexDef, ctx); } + void UpdateIndex(const IndexDef &indexDef, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::UpdateIndex>(indexDef, ctx); } + void DropIndex(const IndexDef &indexDef, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::DropIndex>(indexDef, ctx); } + void SetSchema(std::string_view schema, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::SetSchema>(schema, ctx); } + std::string GetSchema(int format, const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::GetSchema>(format, ctx); } + std::shared_ptr GetSchemaPtr(const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::GetSchemaPtr>(ctx); } + void Insert(Item &item, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::Insert>(item, ctx); } void Insert(Item &item, QueryResults &qr, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeInsert>(item, qr, ctx); } - void Update(Item &item, const RdxContext &ctx) { - nsFuncWrapper(item, ctx); - } + void Update(Item &item, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::Update>(item, ctx); } void Update(Item &item, QueryResults &qr, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeUpdate>(item, qr, ctx); } void Update(const Query &query, QueryResults &result, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::doUpdate, QueryType::QueryUpdate>(query, result, ctx); } - void Upsert(Item &item, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Upsert)(item, ctx); } + void Upsert(Item &item, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::Upsert>(item, ctx); } void Upsert(Item &item, QueryResults &qr, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeUpsert>(item, qr, ctx); } - void Delete(Item &item, const RdxContext &ctx) { - nsFuncWrapper(item, ctx); - } + void Delete(Item &item, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::Delete>(item, ctx); } void Delete(Item &item, QueryResults &qr, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeDelete>(item, qr, ctx); } void Delete(const Query &query, QueryResults &result, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::doDelete, QueryType::QueryDelete>(query, result, ctx); } - void Truncate(const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Truncate)(ctx); } + void Truncate(const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::Truncate>(ctx); } void Select(QueryResults &result, SelectCtx ¶ms, const RdxContext &ctx) { - handleInvalidation(NamespaceImpl::Select)(result, params, ctx); + nsFuncWrapper<&NamespaceImpl::Select>(result, params, ctx); } - NamespaceDef GetDefinition(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetDefinition)(ctx); } - NamespaceMemStat GetMemStat(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetMemStat)(ctx); } + NamespaceDef GetDefinition(const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::GetDefinition>(ctx); } + NamespaceMemStat GetMemStat(const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::GetMemStat>(ctx); } NamespacePerfStat GetPerfStat(const RdxContext &ctx); void ResetPerfStat(const RdxContext &ctx) { txStatsCounter_.Reset(); commitStatsCounter_.Reset(); copyStatsCounter_.Reset(); - handleInvalidation(NamespaceImpl::ResetPerfStat)(ctx); + nsFuncWrapper<&NamespaceImpl::ResetPerfStat>(ctx); } - std::vector EnumMeta(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::EnumMeta)(ctx); } + std::vector EnumMeta(const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::EnumMeta>(ctx); } void BackgroundRoutine(RdxActivityContext *ctx) { if (hasCopy_.load(std::memory_order_acquire)) { return; } - handleInvalidation(NamespaceImpl::BackgroundRoutine)(ctx); + nsFuncWrapper<&NamespaceImpl::BackgroundRoutine>(ctx); } void StorageFlushingRoutine() { if (hasCopy_.load(std::memory_order_acquire)) { return; } - handleInvalidation(NamespaceImpl::StorageFlushingRoutine)(); + nsFuncWrapper<&NamespaceImpl::StorageFlushingRoutine>(); } - void CloseStorage(const RdxContext &ctx) { handleInvalidation(NamespaceImpl::CloseStorage)(ctx); } - Transaction NewTransaction(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::NewTransaction)(ctx); } + void CloseStorage(const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::CloseStorage>(ctx); } + Transaction NewTransaction(const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::NewTransaction>(ctx); } - Item NewItem(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::NewItem)(ctx); } - void ToPool(ItemImpl *item) { handleInvalidation(NamespaceImpl::ToPool)(item); } - std::string GetMeta(const std::string &key, const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetMeta)(key, ctx); } + Item NewItem(const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::NewItem>(ctx); } + void ToPool(ItemImpl *item) { nsFuncWrapper<&NamespaceImpl::ToPool>(item); } + std::string GetMeta(const std::string &key, const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::GetMeta>(key, ctx); } void PutMeta(const std::string &key, std::string_view data, const RdxContext &ctx) { - handleInvalidation(NamespaceImpl::PutMeta)(key, data, ctx); + nsFuncWrapper<&NamespaceImpl::PutMeta>(key, data, ctx); } - int64_t GetSerial(const std::string &field) { return handleInvalidation(NamespaceImpl::GetSerial)(field); } - int getIndexByName(std::string_view index) const { - return nsFuncWrapper(index); - } - bool getIndexByName(std::string_view name, int &index) const { - return nsFuncWrapper(name, index); - } - void FillResult(QueryResults &result, const IdSet &ids) const { handleInvalidation(NamespaceImpl::FillResult)(result, ids); } - void EnablePerfCounters(bool enable = true) { handleInvalidation(NamespaceImpl::EnablePerfCounters)(enable); } - ReplicationState GetReplState(const RdxContext &ctx) const { return handleInvalidation(NamespaceImpl::GetReplState)(ctx); } - void SetReplLSNs(LSNPair LSNs, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::SetReplLSNs)(LSNs, ctx); } + int64_t GetSerial(const std::string &field) { return nsFuncWrapper<&NamespaceImpl::GetSerial>(field); } + int getIndexByName(std::string_view index) const { return nsFuncWrapper<&NamespaceImpl::getIndexByName>(index); } + bool getIndexByName(std::string_view name, int &index) const { return nsFuncWrapper<&NamespaceImpl::tryGetIndexByName>(name, index); } + void FillResult(QueryResults &result, const IdSet &ids) const { nsFuncWrapper<&NamespaceImpl::FillResult>(result, ids); } + void EnablePerfCounters(bool enable = true) { nsFuncWrapper<&NamespaceImpl::EnablePerfCounters>(enable); } + ReplicationState GetReplState(const RdxContext &ctx) const { return nsFuncWrapper<&NamespaceImpl::GetReplState>(ctx); } + void SetReplLSNs(LSNPair LSNs, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::SetReplLSNs>(LSNs, ctx); } void SetSlaveReplStatus(ReplicationState::Status status, const Error &error, const RdxContext &ctx) { - handleInvalidation(NamespaceImpl::SetSlaveReplStatus)(status, error, ctx); + nsFuncWrapper<&NamespaceImpl::SetSlaveReplStatus>(status, error, ctx); } void SetSlaveReplMasterState(MasterState state, const RdxContext &ctx) { - handleInvalidation(NamespaceImpl::SetSlaveReplMasterState)(state, ctx); + nsFuncWrapper<&NamespaceImpl::SetSlaveReplMasterState>(state, ctx); } Error ReplaceTagsMatcher(const TagsMatcher &tm, const RdxContext &ctx) { - return handleInvalidation(NamespaceImpl::ReplaceTagsMatcher)(tm, ctx); + return nsFuncWrapper<&NamespaceImpl::ReplaceTagsMatcher>(tm, ctx); } void Rename(const Namespace::Ptr &dst, const std::string &storagePath, const RdxContext &ctx) { if (this == dst.get() || dst == nullptr) { @@ -142,25 +203,28 @@ class Namespace { txSizeToAlwaysCopy_.store(configData.txSizeToAlwaysCopy, std::memory_order_relaxed); longTxLoggingParams_.store(configProvider.GetTxLoggingParams(), std::memory_order_relaxed); longUpdDelLoggingParams_.store(configProvider.GetUpdDelLoggingParams(), std::memory_order_relaxed); - handleInvalidation(NamespaceImpl::OnConfigUpdated)(configProvider, ctx); + nsFuncWrapper<&NamespaceImpl::OnConfigUpdated>(configProvider, ctx); + } + StorageOpts GetStorageOpts(const RdxContext &ctx) { return nsFuncWrapper<&NamespaceImpl::GetStorageOpts>(ctx); } + void Refill(std::vector &items, const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::Refill>(items, ctx); } + void SetTagsMatcher(TagsMatcher &&tm, const RdxContext &ctx) { + return nsFuncWrapper<&NamespaceImpl::SetTagsMatcher>(std::move(tm), ctx); } - StorageOpts GetStorageOpts(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetStorageOpts)(ctx); } - void Refill(std::vector &items, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Refill)(items, ctx); } void DumpIndex(std::ostream &os, std::string_view index, const RdxContext &ctx) { - return handleInvalidation(NamespaceImpl::DumpIndex)(os, index, ctx); + return nsFuncWrapper<&NamespaceImpl::DumpIndex>(os, index, ctx); } - void SetDestroyFlag() { return handleInvalidation(NamespaceImpl::SetDestroyFlag)(); } + void SetDestroyFlag() { return nsFuncWrapper<&NamespaceImpl::SetDestroyFlag>(); } protected: friend class ReindexerImpl; friend class QueryResults; - void updateSelectTime() const { handleInvalidation(NamespaceImpl::updateSelectTime)(); } - void setSlaveMode(const RdxContext &ctx) { handleInvalidation(NamespaceImpl::setSlaveMode)(ctx); } + void updateSelectTime() const { nsFuncWrapper<&NamespaceImpl::updateSelectTime>(); } + void setSlaveMode(const RdxContext &ctx) { nsFuncWrapper<&NamespaceImpl::setSlaveMode>(ctx); } NamespaceImpl::Ptr getMainNs() const { return atomicLoadMainNs(); } NamespaceImpl::Ptr awaitMainNs(const RdxContext &ctx) const { if (hasCopy_.load(std::memory_order_acquire)) { - contexted_unique_lock lck(clonerMtx_, &ctx); + contexted_unique_lock lck(clonerMtx_, ctx); assertrx(!hasCopy_.load(std::memory_order_acquire)); return ns_; } @@ -168,77 +232,6 @@ class Namespace { } private: - template - typename std::invoke_result::type nsFuncWrapper(Args &&...args) const { - while (true) { - try { - auto ns = atomicLoadMainNs(); - return (*ns.*fn)(std::forward(args)...); - } catch (const Error &e) { - if (e.code() != errNamespaceInvalidated) { - throw; - } else { - std::this_thread::yield(); - } - } - } - } - - template - void nsFuncWrapper(Item &item, QueryResults &qr, NsContext ctx) const { - nsFuncWrapper(item, qr, ctx); - } - - template - void nsFuncWrapper(const Query &query, QueryResults &qr, NsContext ctx) const { - nsFuncWrapper(query, qr, - ctx); - } - - template , ItemModifyMode, QueryType> enumVal> - void nsFuncWrapper(T &v, QueryResults &qr, NsContext ctx) const { - while (true) { - NamespaceImpl::Ptr ns; - bool added = false; - try { - ns = atomicLoadMainNs(); - PerfStatCalculatorMT calc(ns->updatePerfCounter_, ns->enablePerfCounters_); - CounterGuardAIR32 cg(ns->cancelCommitCnt_); - if constexpr (std::is_same_v) { - auto locker{ns->wLock(ctx.rdxContext)}; - calc.LockHit(); - cg.Reset(); - ns->checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); - qr.AddNamespace(ns, true); - added = true; - (*ns.*fn)(v, enumVal, ctx); - qr.AddItem(v, true, false); - ns->tryForceFlush(std::move(locker)); - } else if constexpr (std::is_same_v) { - auto params = longUpdDelLoggingParams_.load(std::memory_order_relaxed); - const bool isEnabled = params.thresholdUs >= 0 && !isSystemNamespaceNameFast(v.NsName()); - auto statCalculator = QueryStatCalculator(long_actions::MakeLogger(v, std::move(params)), isEnabled); - auto locker = statCalculator.CreateLock(*ns, &NamespaceImpl::wLock, ctx.rdxContext); - calc.LockHit(); - cg.Reset(); - ns->checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); - qr.AddNamespace(ns, true); - added = true; - (*ns.*fn)(v, qr, ctx); - statCalculator.LogFlushDuration(*ns, &NamespaceImpl::tryForceFlush, std::move(locker)); - } - return; - } catch (const Error &e) { - if (e.code() != errNamespaceInvalidated) { - throw; - } else { - if (added) qr.RemoveNamespace(ns.get()); - std::this_thread::yield(); - } - } - } - } - bool needNamespaceCopy(const NamespaceImpl::Ptr &ns, const Transaction &tx) const noexcept; void doRename(const Namespace::Ptr &dst, const std::string &newName, const std::string &storagePath, const RdxContext &ctx); NamespaceImpl::Ptr atomicLoadMainNs() const { @@ -267,6 +260,4 @@ class Namespace { BackgroundNamespaceDeleter &bgDeleter_; }; -#undef handleInvalidation - } // namespace reindexer diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index ef58a7116..592638944 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -358,8 +358,10 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes : private RollBackBase { if (idx->HoldsStrings()) { ns_.strHolder_->Add(std::move(idx)); } + // NOLINTBEGIN(bugprone-empty-catch) } catch (...) { } + // NOLINTEND(bugprone-empty-catch) } } void RollBack() noexcept { @@ -657,6 +659,7 @@ void NamespaceImpl::SetSchema(std::string_view schema, const RdxContext& ctx) { } } + const auto initTmVer = tagsMatcher_.version(); schema_ = std::make_shared(schema); auto fields = schema_->GetPaths(); for (auto& field : fields) { @@ -667,6 +670,19 @@ void NamespaceImpl::SetSchema(std::string_view schema, const RdxContext& ctx) { saveSchemaToStorage(); addToWAL(schema, WalSetSchema, ctx); + if (initTmVer != tagsMatcher_.version()) { + const lsn_t lsn(wal_.Add(WALRecord(WalEmpty), lsn_t()), serverId_); + if (!ctx.fromReplication_) repl_.lastSelfLSN = lsn; + if (!repl_.temporary) { + WrSerializer wser; + wser.PutVarint(tagsMatcher_.version()); + wser.PutVarint(tagsMatcher_.stateToken()); + tagsMatcher_.serialize(wser); + // This record is matter for the online replication only + observers_->OnWALUpdate(LSNPair(lsn, ctx.fromReplication_ ? ctx.LSNs_.originLSN_ : lsn), name_, + WALRecord(WalTagsMatcher, wser.Slice())); + } + } } std::string NamespaceImpl::GetSchema(int format, const RdxContext& ctx) { @@ -789,7 +805,7 @@ void NamespaceImpl::verifyCompositeIndex(const IndexDef& indexDef) const { } for (const auto& jp : indexDef.jsonPaths_) { int idx; - if (!getIndexByName(jp, idx)) { + if (!tryGetIndexByName(jp, idx)) { if (!IsFullText(indexDef.Type())) { throw Error(errParams, "Composite indexes over non-indexed field ('%s') are not supported yet (except for full-text indexes). Create " @@ -924,14 +940,18 @@ class NamespaceImpl::RollBack_insertIndex : private RollBackBase { if (insertedIdxName_) { try { ns_.indexesNames_.erase(*insertedIdxName_); + // NOLINTBEGIN(bugprone-empty-catch) } catch (...) { } + // NOLINTEND(bugprone-empty-catch) } if (pkIndexNameInserted_) { try { ns_.indexesNames_.erase(kPKIndexName); + // NOLINTBEGIN(bugprone-empty-catch) } catch (...) { } + // NOLINTEND(bugprone-empty-catch) } for (auto& n : ns_.indexesNames_) { if (n.second > insertedIdxNo_) { @@ -940,8 +960,10 @@ class NamespaceImpl::RollBack_insertIndex : private RollBackBase { } try { ns_.indexes_.erase(insertedIndex_); + // NOLINTBEGIN(bugprone-empty-catch) } catch (...) { } + // NOLINTEND(bugprone-empty-catch) Disable(); } void PkIndexNameInserted() noexcept { pkIndexNameInserted_ = true; } @@ -1219,7 +1241,7 @@ int NamespaceImpl::getIndexByName(std::string_view index) const { int NamespaceImpl::getIndexByNameOrJsonPath(std::string_view index) const { int idx; - if (getIndexByName(index, idx)) { + if (tryGetIndexByName(index, idx)) { return idx; } idx = payloadType_.FieldByJsonPath(index); @@ -1232,7 +1254,7 @@ int NamespaceImpl::getIndexByNameOrJsonPath(std::string_view index) const { int NamespaceImpl::getScalarIndexByName(std::string_view index) const { int idx; - if (getIndexByName(index, idx)) { + if (tryGetIndexByName(index, idx)) { if (idx < indexes_.firstCompositePos()) { return idx; } @@ -1240,7 +1262,7 @@ int NamespaceImpl::getScalarIndexByName(std::string_view index) const { throw Error(errParams, "Index '%s' not found in '%s'", index, name_); } -bool NamespaceImpl::getIndexByName(std::string_view name, int& index) const { +bool NamespaceImpl::tryGetIndexByName(std::string_view name, int& index) const { auto it = indexesNames_.find(name); if (it == indexesNames_.end()) return false; index = it->second; @@ -1248,7 +1270,7 @@ bool NamespaceImpl::getIndexByName(std::string_view name, int& index) const { } bool NamespaceImpl::getIndexByNameOrJsonPath(std::string_view name, int& index) const { - if (getIndexByName(name, index)) { + if (tryGetIndexByName(name, index)) { return true; } const auto idx = payloadType_.FieldByJsonPath(name); @@ -1261,7 +1283,7 @@ bool NamespaceImpl::getIndexByNameOrJsonPath(std::string_view name, int& index) bool NamespaceImpl::getScalarIndexByName(std::string_view name, int& index) const { int idx; - if (getIndexByName(name, idx)) { + if (tryGetIndexByName(name, idx)) { if (idx < indexes_.firstCompositePos()) { index = idx; return true; @@ -1698,20 +1720,41 @@ void NamespaceImpl::CommitTransaction(Transaction& tx, QueryResults& result, NsC storageAdvice = storage_.AdviceBatching(); } - for (auto& step : tx.GetSteps()) { - if (step.query_) { - QueryResults qr; - qr.AddNamespace(this, true); - if (step.query_->type_ == QueryDelete) { - doDelete(*step.query_, qr, ctx); - } else { - doUpdate(*step.query_, qr, ctx); + for (auto&& step : tx.GetSteps()) { + switch (step.type_) { + case TransactionStep::Type::ModifyItem: { + const auto mode = std::get(step.data_).mode; + Item item = tx.GetItem(std::move(step)); + modifyItem(item, mode, ctx); + result.AddItem(item); + break; } - } else { - const auto modifyMode = step.modifyMode_; - Item item = tx.GetItem(std::move(step)); - modifyItem(item, modifyMode, ctx); - result.AddItem(item); + case TransactionStep::Type::Query: { + QueryResults qr; + qr.AddNamespace(this, true); + auto& data = std::get(step.data_); + if (data.query->type_ == QueryDelete) { + doDelete(*data.query, qr, ctx); + } else { + doUpdate(*data.query, qr, ctx); + } + break; + } + case TransactionStep::Type::Nop: + break; + case TransactionStep::Type::PutMeta: { + auto& data = std::get(step.data_); + putMeta(data.key, data.value, ctx.rdxContext); + break; + } + case TransactionStep::Type::SetTM: { + auto& data = std::get(step.data_); + auto tmCopy = data.tm; + setTagsMatcher(std::move(tmCopy), ctx); + break; + } + default: + std::abort(); } } @@ -1810,7 +1853,7 @@ void NamespaceImpl::doUpsert(ItemImpl* ritem, IdType id, bool doUpdate) { } else { pl.Get(field, krefs, index.Opts().IsArray()); } - if (krefs == skrefs) continue; + if ((krefs.ArrayType().Is() && skrefs.ArrayType().Is()) || krefs == skrefs) continue; bool needClearCache{false}; index.Delete(krefs, id, *strHolder_, needClearCache); if (needClearCache && index.IsOrdered()) indexesCacheCleaner.Add(index.SortId()); @@ -2567,6 +2610,11 @@ std::shared_ptr NamespaceImpl::GetSchemaPtr(const RdxContext& ctx) return schema_; } +void NamespaceImpl::SetTagsMatcher(TagsMatcher&& tm, const RdxContext& ctx) { + auto wlck = wLock(ctx); + setTagsMatcher(std::move(tm), ctx); +} + void NamespaceImpl::LoadFromStorage(unsigned threadsCount, const RdxContext& ctx) { auto wlck = wLock(ctx); FlagGuardT nsLoadingGuard(nsIsLoading_); @@ -2647,6 +2695,31 @@ void NamespaceImpl::removeExpiredStrings(RdxActivityContext* ctx) { } } +void NamespaceImpl::setTagsMatcher(TagsMatcher&& tm, const NsContext& ctx) { + // NOTE: In v4 tm tokens here are always the same, but in v3 those tokens are not synchronized. Probably it should workd anyway + // if (tm.stateToken() != tagsMatcher_.stateToken()) { + // throw Error(errParams, "Tagsmatcher have different statetokens: %08X vs %08X", tagsMatcher_.stateToken(), tm.stateToken()); + // } + if (!ctx.rdxContext.fromReplication_) { + throw Error(errParams, "Tagsmatcher can be set from replication only"); + } + tagsMatcher_ = tm; + tagsMatcher_.UpdatePayloadType(payloadType_, false); + tagsMatcher_.setUpdated(); + + const lsn_t lsn(wal_.Add(WALRecord(WalEmpty, 0, ctx.inTransaction)), serverId_); + if (!repl_.temporary) { + WrSerializer ser; + ser.PutVarint(tagsMatcher_.version()); + ser.PutVarint(tagsMatcher_.stateToken()); + tagsMatcher_.serialize(ser); + observers_->OnWALUpdate(LSNPair(lsn, ctx.rdxContext.LSNs_.originLSN_), name_, + WALRecord(WalTagsMatcher, ser.Slice(), ctx.inTransaction)); + } + + saveTagsMatcherToStorage(true); +} + void NamespaceImpl::BackgroundRoutine(RdxActivityContext* ctx) { const RdxContext rdxCtx(ctx); const NsContext nsCtx(rdxCtx); diff --git a/cpp_src/core/namespace/namespaceimpl.h b/cpp_src/core/namespace/namespaceimpl.h index 31a6e8fb4..967349477 100644 --- a/cpp_src/core/namespace/namespaceimpl.h +++ b/cpp_src/core/namespace/namespaceimpl.h @@ -138,7 +138,7 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. friend class ItemsLoader; friend class IndexInserters; - class NSUpdateSortedContext : public UpdateSortedContext { + class NSUpdateSortedContext final : public UpdateSortedContext { public: NSUpdateSortedContext(const NamespaceImpl &ns, SortType curSortId) : ns_(ns), sorted_indexes_(ns_.getSortedIdxCount()), curSortId_(curSortId) { @@ -259,7 +259,7 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. int getIndexByName(std::string_view index) const; int getIndexByNameOrJsonPath(std::string_view name) const; int getScalarIndexByName(std::string_view name) const; - bool getIndexByName(std::string_view name, int &index) const; + bool tryGetIndexByName(std::string_view name, int &index) const; bool getIndexByNameOrJsonPath(std::string_view name, int &index) const; bool getScalarIndexByName(std::string_view name, int &index) const; bool getSparseIndexByJsonPath(std::string_view jsonPath, int &index) const; @@ -282,6 +282,9 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. std::shared_ptr GetSchemaPtr(const RdxContext &ctx) const; int getNsNumber() const { return schema_ ? schema_->GetProtobufNsNumber() : 0; } IndexesCacheCleaner GetIndexesCacheCleaner() { return IndexesCacheCleaner{*this}; } + // Separate method for the v3/v4 replication compatibility. + // It should not be used outside of this scenario + void SetTagsMatcher(TagsMatcher &&tm, const RdxContext &ctx); void SetDestroyFlag() { dbDestroyed_ = true; } Error FlushStorage(const RdxContext &ctx) { const auto flushOpts = StorageFlushOpts().WithImmediateReopen(); @@ -303,10 +306,10 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. typedef contexted_shared_lock RLockT; typedef contexted_unique_lock WLockT; - RLockT RLock(const RdxContext &ctx) const { return RLockT(mtx_, &ctx); } + RLockT RLock(const RdxContext &ctx) const { return RLockT(mtx_, ctx); } WLockT WLock(const RdxContext &ctx) const { using namespace std::string_view_literals; - WLockT lck(mtx_, &ctx); + WLockT lck(mtx_, ctx); if (readonly_.load(std::memory_order_acquire)) { throw Error(errNamespaceInvalidated, "NS invalidated"sv); } @@ -373,6 +376,7 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. std::optional &&modifyData); void removeExpiredItems(RdxActivityContext *); void removeExpiredStrings(RdxActivityContext *); + void setTagsMatcher(TagsMatcher &&tm, const NsContext &ctx); Item newItem(); template diff --git a/cpp_src/core/nsselecter/explaincalc.cc b/cpp_src/core/nsselecter/explaincalc.cc index 484efe308..419ad12d3 100644 --- a/cpp_src/core/nsselecter/explaincalc.cc +++ b/cpp_src/core/nsselecter/explaincalc.cc @@ -192,6 +192,7 @@ std::string ExplainCalc::GetJSON() { WrSerializer ser; { JsonBuilder json(ser); + json.EmitTrailingForFloat(false); if (enabled_) { json.Put("total_us"sv, To_us(total_)); json.Put("preselect_us"sv, To_us(preselect_)); diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index 937707fb9..f2ec42bfb 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -65,10 +65,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte } OnConditionInjections explainInjectedOnConditions; - QueryPreprocessor qPreproc((ctx.preResult && ctx.preResult->executionMode == JoinPreResult::ModeExecute) - ? const_cast(&ctx.query.entries)->MakeLazyCopy() - : QueryEntries{ctx.query.entries}, - ns_, ctx); + QueryPreprocessor qPreproc(QueryEntries{ctx.query.entries}, ns_, ctx); if (ctx.joinedSelectors) { qPreproc.InjectConditionsFromJoins(*ctx.joinedSelectors, explainInjectedOnConditions, rdxCtx); explain.PutOnConditionInjections(&explainInjectedOnConditions); @@ -182,7 +179,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte qres.Append(OpAnd, SelectIterator(std::move(res), false, pr, IteratorFieldKind::None)); } break; case JoinPreResult::ModeIterators: - qres.LazyAppend(ctx.preResult->iterators.begin(), ctx.preResult->iterators.end()); + qres.Append(ctx.preResult->iterators.begin(), ctx.preResult->iterators.end()); break; case JoinPreResult::ModeValues: assertrx(0); @@ -627,65 +624,64 @@ class ForcedSortMap { static_cast(*this)); } const auto &operator*() const { - return std::visit(overloaded{[](MultiMap::Iterator it) -> const auto &{ return *it; - } - , [](SingleTypeMap::const_iterator it) -> const auto & { return *it; }}, + return std::visit(overloaded{[](MultiMap::Iterator it) -> const auto & { return *it; }, + [](SingleTypeMap::const_iterator it) -> const auto & { return *it; }}, static_cast(*this)); -} -}; // namespace reindexer + } + }; // namespace reindexer public: -ForcedSortMap(Variant k, mapped_type v, size_t size) - : data_{k.Type().Is() || k.Type().Is() || k.Type().IsNumeric() - ? DataType{MultiMap{size}} - : DataType{SingleTypeMap{{}, k.Type()}}} { - std::visit(overloaded{[&](MultiMap &m) { m.insert(std::move(k), v); }, [&](SingleTypeMap &m) { m.emplace(std::move(k), v); }}, data_); -} -std::pair emplace(Variant k, mapped_type v) & { - return std::visit(overloaded{[&](MultiMap &m) { - const auto [iter, success] = m.insert(std::move(k), v); - return std::make_pair(Iterator{iter}, success); - }, - [&](SingleTypeMap &m) { - if (!m.type_.IsSame(k.Type())) { - throw Error{errQueryExec, "Items of different types in forced sort list"}; - } - const auto [iter, success] = m.emplace(std::move(k), v); - return std::make_pair(Iterator{iter}, success); - }}, - data_); -} -bool contain(const Variant &k) const { - return std::visit(overloaded{[&k](const MultiMap &m) { return m.find(k) != m.cend(); }, - [&k](const SingleTypeMap &m) { - if (!m.type_.IsSame(k.Type())) { - throw Error{errQueryExec, "Items of different types in forced sort list"}; - } - return m.find(k) != m.end(); - }}, - data_); -} -mapped_type get(const Variant &k) const { - return std::visit(overloaded{[&k](const MultiMap &m) { - const auto it = m.find(k); - assertrx_throw(it != m.cend()); - return it->second; - }, - [&k](const SingleTypeMap &m) { - if (!m.type_.IsSame(k.Type())) { - throw Error{errQueryExec, "Items of different types in forced sort list"}; - } - const auto it = m.find(k); - assertrx_throw(it != m.end()); - return it->second; - }}, - data_); -} + ForcedSortMap(Variant k, mapped_type v, size_t size) + : data_{k.Type().Is() || k.Type().Is() || k.Type().IsNumeric() + ? DataType{MultiMap{size}} + : DataType{SingleTypeMap{{}, k.Type()}}} { + std::visit(overloaded{[&](MultiMap &m) { m.insert(std::move(k), v); }, [&](SingleTypeMap &m) { m.emplace(std::move(k), v); }}, + data_); + } + std::pair emplace(Variant k, mapped_type v) & { + return std::visit(overloaded{[&](MultiMap &m) { + const auto [iter, success] = m.insert(std::move(k), v); + return std::make_pair(Iterator{iter}, success); + }, + [&](SingleTypeMap &m) { + if (!m.type_.IsSame(k.Type())) { + throw Error{errQueryExec, "Items of different types in forced sort list"}; + } + const auto [iter, success] = m.emplace(std::move(k), v); + return std::make_pair(Iterator{iter}, success); + }}, + data_); + } + bool contain(const Variant &k) const { + return std::visit(overloaded{[&k](const MultiMap &m) { return m.find(k) != m.cend(); }, + [&k](const SingleTypeMap &m) { + if (!m.type_.IsSame(k.Type())) { + throw Error{errQueryExec, "Items of different types in forced sort list"}; + } + return m.find(k) != m.end(); + }}, + data_); + } + mapped_type get(const Variant &k) const { + return std::visit(overloaded{[&k](const MultiMap &m) { + const auto it = m.find(k); + assertrx_throw(it != m.cend()); + return it->second; + }, + [&k](const SingleTypeMap &m) { + if (!m.type_.IsSame(k.Type())) { + throw Error{errQueryExec, "Items of different types in forced sort list"}; + } + const auto it = m.find(k); + assertrx_throw(it != m.end()); + return it->second; + }}, + data_); + } private: -DataType data_; -} -; + DataType data_; +}; template class ForcedMapInserter { @@ -1040,7 +1036,7 @@ void NsSelecter::selectLoop(LoopCtx &ctx, ResultsT &result, const RdxContext &rd // Exclude last sets of id from each query result, so duplicated keys will // be removed for (auto &it : qres) { - if (it.HoldsOrReferTo() && it.Value().distinct) { + if (it.Is() && it.Value().distinct) { it.Value().ExcludeLastSet(pv, rowId, properRowId); } } @@ -1214,7 +1210,7 @@ void NsSelecter::addSelectResult(uint8_t proc, IdType rowId, IdType properRowId, void NsSelecter::checkStrictModeAgg(StrictMode strictMode, const std::string &name, const std::string &nsName, const TagsMatcher &tagsMatcher) const { - if (int index = IndexValueType::SetByJsonPath; ns_->getIndexByName(name, index)) return; + if (int index = IndexValueType::SetByJsonPath; ns_->tryGetIndexByName(name, index)) return; if (strictMode == StrictModeIndexes) { throw Error(errParams, @@ -1659,7 +1655,7 @@ bool NsSelecter::isSortOptimizatonEffective(const QueryEntries &qentries, Select if (qentries.Size() == 0) { return true; } - if (qentries.Size() == 1 && qentries.HoldsOrReferTo(0)) { + if (qentries.Size() == 1 && qentries.Is(0)) { const auto &qe = qentries.Get(0); if (qe.IndexNo() == ctx.sortingContext.uncommitedIndex) { return SelectIteratorContainer::IsExpectingOrderedResults(qe); diff --git a/cpp_src/core/nsselecter/querypreprocessor.cc b/cpp_src/core/nsselecter/querypreprocessor.cc index 6070eb478..e18a8a0ff 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.cc +++ b/cpp_src/core/nsselecter/querypreprocessor.cc @@ -54,7 +54,7 @@ void QueryPreprocessor::ExcludeFtQuery(const RdxContext &rdxCtx) { if (queryEntryAddedByForcedSortOptimization_ || Size() <= 1) return; for (auto it = begin(), next = it, endIt = end(); it != endIt; it = next) { ++next; - if (it->HoldsOrReferTo() && it->Value().IsFieldIndexed()) { + if (it->Is() && it->Value().IsFieldIndexed()) { auto &index = ns_.indexes_[it->Value().IndexNo()]; if (!IsFastFullText(index->Type())) continue; if (it->operation != OpAnd || (next != endIt && next->operation == OpOr) || !index->EnablePreselectBeforeFt()) break; @@ -263,7 +263,7 @@ void QueryPreprocessor::CheckUniqueFtQuery() const { bool QueryPreprocessor::ContainsFullTextIndexes() const { for (auto it = cbegin().PlainIterator(), end = cend().PlainIterator(); it != end; ++it) { - if (it->HoldsOrReferTo() && it->Value().IsFieldIndexed() && + if (it->Is() && it->Value().IsFieldIndexed() && IsFullText(ns_.indexes_[it->Value().IndexNo()]->Type())) { return true; } @@ -331,7 +331,7 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si deleted += substituteCompositeIndexes(cur + 1, cur + bracketSize); continue; } - if (!HoldsOrReferTo(cur) || GetOperation(cur) != OpAnd) { + if (!Is(cur) || GetOperation(cur) != OpAnd) { continue; } const auto next = Next(cur); @@ -484,10 +484,6 @@ bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { if rx_unlikely (lqe->Values().empty()) { return true; } - if (container_[lhs].IsRef()) { - container_[lhs].SetValue(const_cast(*lqe)); - lqe = &Get(lhs); - } const bool distinct = lqe->Distinct() || rqe.Distinct(); VariantArray setValues; if (rx_likely(!rqe.Values().empty())) { @@ -501,7 +497,7 @@ bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { constexpr size_t kMinArraySizeToUseHashSet = 250; if (second.size() < kMinArraySizeToUseHashSet) { // Intersect via binary search + sort for small vectors - std::sort(first.begin(), first.end()); + boost::sort::pdqsort(first.begin(), first.end()); for (auto &&v : second) { if (std::binary_search(first.begin(), first.end(), v)) { setValues.emplace_back(std::move(v)); @@ -527,20 +523,12 @@ bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { return true; } else if (rqe.Condition() == CondAny) { if (!lqe->Distinct() && rqe.Distinct()) { - if (container_[lhs].IsRef()) { - container_[lhs].SetValue(const_cast(*lqe)); - lqe = &Get(lhs); - } lqe->Distinct(true); } return true; } else if (lqe->Condition() == CondAny) { const bool distinct = lqe->Distinct() || rqe.Distinct(); - if (container_[rhs].IsRef()) { - container_[lhs].SetValue(const_cast(rqe)); - } else { - container_[lhs].SetValue(std::move(rqe)); - } + container_[lhs].SetValue(std::move(rqe)); Get(lhs).Distinct(distinct); return true; } @@ -838,7 +826,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join CondType queryCondition{CondAny}; VariantArray values; if (byValues) { - assertrx_throw(joinedSelector.itemQuery_.entries.HoldsOrReferTo(i)); + assertrx_throw(joinedSelector.itemQuery_.entries.Is(i)); assertrx_throw(joinedSelector.itemQuery_.entries.Get(i).FieldName() == joinEntry.RightFieldName()); CollateOpts collate; if (joinEntry.IsLeftFieldIndexed()) { diff --git a/cpp_src/core/nsselecter/querypreprocessor.h b/cpp_src/core/nsselecter/querypreprocessor.h index ba79d4c10..2b12396b4 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.h +++ b/cpp_src/core/nsselecter/querypreprocessor.h @@ -1,7 +1,6 @@ #pragma once #include "aggregator.h" -#include "core/ft/ftsetcashe.h" #include "core/index/ft_preselect.h" #include "core/query/queryentry.h" #include "estl/h_vector.h" diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.cc b/cpp_src/core/nsselecter/selectiteratorcontainer.cc index f34f73137..1772e9daa 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.cc +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.cc @@ -144,7 +144,7 @@ double SelectIteratorContainer::fullCost(span indexes, unsigned cur, u } bool SelectIteratorContainer::isIdset(const_iterator it, const_iterator end) { - return it->operation == OpAnd && it->HoldsOrReferTo() && + return it->operation == OpAnd && it->Is() && it->Value().comparators_.empty() && // !it->Value().empty() && (++it == end || it->operation != OpOr); } @@ -176,10 +176,9 @@ void SelectIteratorContainer::CheckFirstQuery() { // Let iterators choose most effective algorithm void SelectIteratorContainer::SetExpectMaxIterations(int expectedIterations) { assertrx(!Empty()); - assertrx(HoldsOrReferTo(0)); + assertrx(Is(0)); for (Container::iterator it = container_.begin() + 1; it != container_.end(); ++it) { - if (it->HoldsOrReferTo()) { - if (it->IsRef()) it->SetValue(it->Value()); + if (it->Is()) { it->Value().SetExpectMaxIterations(expectedIterations); } } @@ -308,10 +307,7 @@ void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectR if (last == this->end()) { throw Error(errQueryExec, "OR operator in first condition or after left join"); } - if (last->HoldsOrReferTo() && !last->Value().distinct && last->operation != OpNot) { - if (last->IsRef()) { - last->SetValue(last->Value()); - } + if (last->Is() && !last->Value().distinct && last->operation != OpNot) { SelectIterator &it = last->Value(); if (!qe.IsFieldIndexed() || isIndexSparse) { it.Append(res); @@ -331,9 +327,6 @@ void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectR if (qe.IsFieldIndexed() && !isIndexSparse) { // last appended is always a SelectIterator const auto lastAppendedIt = lastAppendedOrClosed(); - if (lastAppendedIt->IsRef()) { - lastAppendedIt->SetValue(lastAppendedIt->Value()); - } SelectIterator &lastAppended = lastAppendedIt->Value(); lastAppended.Bind(ns.payloadType_, qe.IndexNo()); lastAppended.SetNotOperationFlag(op == OpNot); diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.h b/cpp_src/core/nsselecter/selectiteratorcontainer.h index eb976796d..5c046a0be 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.h +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.h @@ -49,11 +49,11 @@ class SelectIteratorContainer bool IsSelectIterator(size_t i) const noexcept { assertrx(i < Size()); - return container_[i].HoldsOrReferTo(); + return container_[i].Is(); } bool IsJoinIterator(size_t i) const noexcept { assertrx(i < container_.size()); - return container_[i].HoldsOrReferTo(); + return container_[i].Is(); } void ExplainJSON(int iters, JsonBuilder &builder, const std::vector *js) const { explainJSON(cbegin(), cend(), iters, builder, js); diff --git a/cpp_src/core/nsselecter/sortexpression.cc b/cpp_src/core/nsselecter/sortexpression.cc index 6f5c0098a..fb52973d1 100644 --- a/cpp_src/core/nsselecter/sortexpression.cc +++ b/cpp_src/core/nsselecter/sortexpression.cc @@ -76,12 +76,12 @@ VariantArray SortExpression::GetJoinedFieldValues(IdType rowId, const joins::Nam bool SortExpression::ByIndexField() const noexcept { static constexpr SortExpressionOperation noOperation; - return Size() == 1 && container_[0].HoldsOrReferTo() && GetOperation(0) == noOperation; + return Size() == 1 && container_[0].Is() && GetOperation(0) == noOperation; } bool SortExpression::ByJoinedIndexField() const noexcept { static constexpr SortExpressionOperation noOperation; - return Size() == 1 && container_[0].HoldsOrReferTo() && GetOperation(0) == noOperation; + return Size() == 1 && container_[0].Is() && GetOperation(0) == noOperation; } const SortExprFuncs::JoinedIndex& SortExpression::GetJoinedIndex() const noexcept { diff --git a/cpp_src/core/query/query.cc b/cpp_src/core/query/query.cc index 49b6f3dec..48bb915e7 100644 --- a/cpp_src/core/query/query.cc +++ b/cpp_src/core/query/query.cc @@ -526,9 +526,9 @@ void Query::WalkNested(bool withSelf, bool withMerged, const std::function(0) && kLsnIndexName == entries.Get(0).FieldName()) { + if (entries.Size() == 1 && entries.Is(0) && kLsnIndexName == entries.Get(0).FieldName()) { return true; - } else if (entries.Size() == 2 && entries.HoldsOrReferTo(0) && entries.HoldsOrReferTo(1)) { + } else if (entries.Size() == 2 && entries.Is(0) && entries.Is(1)) { const auto &index0 = entries.Get(0).FieldName(); const auto &index1 = entries.Get(1).FieldName(); return (kLsnIndexName == index0 && kSlaveVersionIndexName == index1) || diff --git a/cpp_src/core/query/queryentry.h b/cpp_src/core/query/queryentry.h index 6d00faf35..bde46fa55 100644 --- a/cpp_src/core/query/queryentry.h +++ b/cpp_src/core/query/queryentry.h @@ -48,11 +48,11 @@ class QueryField { [[nodiscard]] int IndexNo() const noexcept { return idxNo_; } [[nodiscard]] bool IsFieldIndexed() const noexcept { return idxNo_ >= 0; } [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return idxNo_ != IndexValueType::NotSet; } - [[nodiscard]] const FieldsSet &Fields() const &noexcept { return fieldsSet_; } - [[nodiscard]] const std::string &FieldName() const &noexcept { return fieldName_; } + [[nodiscard]] const FieldsSet &Fields() const & noexcept { return fieldsSet_; } + [[nodiscard]] const std::string &FieldName() const & noexcept { return fieldName_; } [[nodiscard]] KeyValueType FieldType() const noexcept { return fieldType_; } [[nodiscard]] KeyValueType SelectType() const noexcept { return selectType_; } - [[nodiscard]] const std::vector &CompositeFieldsTypes() const &noexcept { return compositeFieldsTypes_; } + [[nodiscard]] const std::vector &CompositeFieldsTypes() const & noexcept { return compositeFieldsTypes_; } [[nodiscard]] bool HaveEmptyField() const noexcept; void SetField(FieldsSet &&fields) &; void SetIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, @@ -98,9 +98,9 @@ class QueryEntry : private QueryField { verifyIgnoringEmptyValues(); } [[nodiscard]] CondType Condition() const noexcept { return condition_; } - [[nodiscard]] const VariantArray &Values() const &noexcept { return values_; } - [[nodiscard]] VariantArray &&Values() &&noexcept { return std::move(values_); } - [[nodiscard]] auto UpdatableValues(IgnoreEmptyValues) &noexcept { + [[nodiscard]] const VariantArray &Values() const & noexcept { return values_; } + [[nodiscard]] VariantArray &&Values() && noexcept { return std::move(values_); } + [[nodiscard]] auto UpdatableValues(IgnoreEmptyValues) & noexcept { return VerifyingUpdater{*this}; } [[nodiscard]] bool Distinct() const noexcept { return distinct_; } @@ -122,8 +122,8 @@ class QueryEntry : private QueryField { values_ = std::move(values); } - const QueryField &FieldData() const &noexcept { return static_cast(*this); } - QueryField &FieldData() &noexcept { return static_cast(*this); } + const QueryField &FieldData() const & noexcept { return static_cast(*this); } + QueryField &FieldData() & noexcept { return static_cast(*this); } void ConvertValuesToFieldType() & { for (Variant &v : values_) { v.convert(SelectType()); @@ -176,20 +176,20 @@ class BetweenFieldsQueryEntry { [[nodiscard]] CondType Condition() const noexcept { return condition_; } [[nodiscard]] int LeftIdxNo() const noexcept { return leftField_.IndexNo(); } [[nodiscard]] int RightIdxNo() const noexcept { return rightField_.IndexNo(); } - [[nodiscard]] const std::string &LeftFieldName() const &noexcept { return leftField_.FieldName(); } - [[nodiscard]] const std::string &RightFieldName() const &noexcept { return rightField_.FieldName(); } - [[nodiscard]] const FieldsSet &LeftFields() const &noexcept { return leftField_.Fields(); } - [[nodiscard]] const FieldsSet &RightFields() const &noexcept { return rightField_.Fields(); } + [[nodiscard]] const std::string &LeftFieldName() const & noexcept { return leftField_.FieldName(); } + [[nodiscard]] const std::string &RightFieldName() const & noexcept { return rightField_.FieldName(); } + [[nodiscard]] const FieldsSet &LeftFields() const & noexcept { return leftField_.Fields(); } + [[nodiscard]] const FieldsSet &RightFields() const & noexcept { return rightField_.Fields(); } [[nodiscard]] KeyValueType LeftFieldType() const noexcept { return leftField_.FieldType(); } [[nodiscard]] KeyValueType RightFieldType() const noexcept { return rightField_.FieldType(); } - [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const &noexcept { return leftField_.CompositeFieldsTypes(); } - [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const &noexcept { + [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const & noexcept { return leftField_.CompositeFieldsTypes(); } + [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const & noexcept { return rightField_.CompositeFieldsTypes(); } - [[nodiscard]] const QueryField &LeftFieldData() const &noexcept { return leftField_; } - [[nodiscard]] QueryField &LeftFieldData() &noexcept { return leftField_; } - [[nodiscard]] const QueryField &RightFieldData() const &noexcept { return rightField_; } - [[nodiscard]] QueryField &RightFieldData() &noexcept { return rightField_; } + [[nodiscard]] const QueryField &LeftFieldData() const & noexcept { return leftField_; } + [[nodiscard]] QueryField &LeftFieldData() & noexcept { return leftField_; } + [[nodiscard]] const QueryField &RightFieldData() const & noexcept { return rightField_; } + [[nodiscard]] QueryField &RightFieldData() & noexcept { return rightField_; } [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return leftField_.FieldsHaveBeenSet() && rightField_.FieldsHaveBeenSet(); } [[nodiscard]] bool IsLeftFieldIndexed() const noexcept { return leftField_.IsFieldIndexed(); } [[nodiscard]] bool IsRightFieldIndexed() const noexcept { return rightField_.IsFieldIndexed(); } @@ -236,7 +236,6 @@ class QueryEntries QueryEntries(QueryEntries &&) = default; QueryEntries(const QueryEntries &) = default; QueryEntries &operator=(QueryEntries &&) = default; - QueryEntries MakeLazyCopy() & { return {makeLazyCopy()}; } void ToDsl(const Query &parentQuery, JsonBuilder &builder) const { return toDsl(cbegin(), cend(), parentQuery, builder); } void WriteSQLWhere(const Query &parentQuery, WrSerializer &, bool stripArgs) const; @@ -303,23 +302,23 @@ class QueryJoinEntry { [[nodiscard]] bool IsRightFieldIndexed() const noexcept { return rightField_.IsFieldIndexed(); } [[nodiscard]] int LeftIdxNo() const noexcept { return leftField_.IndexNo(); } [[nodiscard]] int RightIdxNo() const noexcept { return rightField_.IndexNo(); } - [[nodiscard]] const FieldsSet &LeftFields() const &noexcept { return leftField_.Fields(); } - [[nodiscard]] const FieldsSet &RightFields() const &noexcept { return rightField_.Fields(); } + [[nodiscard]] const FieldsSet &LeftFields() const & noexcept { return leftField_.Fields(); } + [[nodiscard]] const FieldsSet &RightFields() const & noexcept { return rightField_.Fields(); } [[nodiscard]] KeyValueType LeftFieldType() const noexcept { return leftField_.FieldType(); } [[nodiscard]] KeyValueType RightFieldType() const noexcept { return rightField_.FieldType(); } - [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const &noexcept { return leftField_.CompositeFieldsTypes(); } - [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const &noexcept { + [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const & noexcept { return leftField_.CompositeFieldsTypes(); } + [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const & noexcept { return rightField_.CompositeFieldsTypes(); } [[nodiscard]] OpType Operation() const noexcept { return op_; } [[nodiscard]] CondType Condition() const noexcept { return condition_; } - [[nodiscard]] const std::string &LeftFieldName() const &noexcept { return leftField_.FieldName(); } - [[nodiscard]] const std::string &RightFieldName() const &noexcept { return rightField_.FieldName(); } + [[nodiscard]] const std::string &LeftFieldName() const & noexcept { return leftField_.FieldName(); } + [[nodiscard]] const std::string &RightFieldName() const & noexcept { return rightField_.FieldName(); } [[nodiscard]] bool ReverseNamespacesOrder() const noexcept { return reverseNamespacesOrder_; } - [[nodiscard]] const QueryField &LeftFieldData() const &noexcept { return leftField_; } - [[nodiscard]] QueryField &LeftFieldData() &noexcept { return leftField_; } - [[nodiscard]] const QueryField &RightFieldData() const &noexcept { return rightField_; } - [[nodiscard]] QueryField &RightFieldData() &noexcept { return rightField_; } + [[nodiscard]] const QueryField &LeftFieldData() const & noexcept { return leftField_; } + [[nodiscard]] QueryField &LeftFieldData() & noexcept { return leftField_; } + [[nodiscard]] const QueryField &RightFieldData() const & noexcept { return rightField_; } + [[nodiscard]] QueryField &RightFieldData() & noexcept { return rightField_; } void SetLeftIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, std::vector &&compositeFieldsTypes) & { leftField_.SetIndexData(idxNo, std::move(fields), fieldType, selectType, std::move(compositeFieldsTypes)); diff --git a/cpp_src/core/query/sql/sqlparser.cc b/cpp_src/core/query/sql/sqlparser.cc index 1cb96cfd1..83f3592b5 100644 --- a/cpp_src/core/query/sql/sqlparser.cc +++ b/cpp_src/core/query/sql/sqlparser.cc @@ -767,7 +767,7 @@ void SQLParser::parseEqualPositions(tokenizer &parser, std::vectorHoldsOrReferTo() && nameWithCase.text() == it->Value().FieldName()) { + if (it->Is() && nameWithCase.text() == it->Value().FieldName()) { validField = true; break; } diff --git a/cpp_src/core/query/sql/sqlsuggester.cc b/cpp_src/core/query/sql/sqlsuggester.cc index c8b2cc8da..908e6c930 100644 --- a/cpp_src/core/query/sql/sqlsuggester.cc +++ b/cpp_src/core/query/sql/sqlsuggester.cc @@ -9,7 +9,7 @@ namespace reindexer { -bool checkIfTokenStartsWith(std::string_view src, std::string_view pattern) { +static bool checkIfTokenStartsWith(std::string_view src, std::string_view pattern) { return checkIfStartsWith(src, pattern) && src.length() < pattern.length(); } @@ -22,8 +22,10 @@ std::vector SQLSuggester::GetSuggestions(std::string_view q, size_t try { Parse(q); + // NOLINTBEGIN(bugprone-empty-catch) } catch (const Error &) { } + // NOLINTEND(bugprone-empty-catch) for (SqlParsingCtx::SuggestionData &item : ctx_.suggestions) { checkForTokenSuggestions(item); diff --git a/cpp_src/core/reindexerimpl.cc b/cpp_src/core/reindexerimpl.cc index 59e1f48bd..7babb5b84 100644 --- a/cpp_src/core/reindexerimpl.cc +++ b/cpp_src/core/reindexerimpl.cc @@ -96,7 +96,7 @@ ReindexerImpl::StatsLocker::StatsLocker() { ReindexerImpl::StatsLocker::StatsLockT ReindexerImpl::StatsLocker::LockIfRequired(std::string_view sysNsName, const RdxContext& ctx) { auto found = mtxMap_.find(sysNsName); if (found != mtxMap_.end()) { - return StatsLockT(found->second, &ctx); + return StatsLockT(found->second, ctx); } // Do not create any lock, if namespace does not preset in the map return StatsLockT(); @@ -231,8 +231,9 @@ Error ReindexerImpl::Connect(const std::string& dsn, ConnectOpts opts) { if (!err.ok()) return err; if (enableStorage && opts.IsOpenNamespaces()) { - std::sort(foundNs.begin(), foundNs.end(), - [](const fs::DirEntry& ld, const fs::DirEntry& rd) { return ld.internalFilesCount > rd.internalFilesCount; }); + boost::sort::pdqsort_branchless(foundNs.begin(), foundNs.end(), [](const fs::DirEntry& ld, const fs::DirEntry& rd) noexcept { + return ld.internalFilesCount > rd.internalFilesCount; + }); const size_t maxLoadWorkers = ConcurrentNamespaceLoaders(); std::unique_ptr thrs(new std::thread[maxLoadWorkers]); std::atomic_flag hasNsErrors = ATOMIC_FLAG_INIT; @@ -303,7 +304,7 @@ Error ReindexerImpl::OpenNamespace(std::string_view nsName, const StorageOpts& o const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? (ser << "OPEN NAMESPACE " << nsName).Slice() : ""sv, activities_); { - SLock lock(mtx_, &rdxCtx); + SLock lock(mtx_, rdxCtx); auto it = namespaces_.find(nsName); if (it == namespaces_.end()) { // create new namespace if (!validateUserNsName(nsName)) { @@ -335,7 +336,7 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r Namespace::Ptr ns; try { { - SLock lock(mtx_, &rdxCtx); + SLock lock(mtx_, rdxCtx); if (namespaces_.find(nsDef.name) != namespaces_.end()) { return Error(errParams, "Namespace '%s' already exists", nsDef.name); } @@ -356,7 +357,7 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r ns->LoadFromStorage(kStorageLoadingThreads, rdxCtx); } { - ULock lock(mtx_, &rdxCtx); + ULock lock(mtx_, rdxCtx); namespaces_.insert({nsDef.name, ns}); } if (!nsDef.isTemporary) observers_.OnWALUpdate(LSNPair(), nsDef.name, WALRecord(WalNamespaceAdd)); @@ -376,7 +377,7 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r Error ReindexerImpl::openNamespace(std::string_view name, const StorageOpts& storageOpts, const RdxContext& rdxCtx) { try { { - SLock lock(mtx_, &rdxCtx); + SLock lock(mtx_, rdxCtx); auto nsIt = namespaces_.find(name); if (nsIt != namespaces_.end() && nsIt->second) { if (storageOpts.IsSlaveMode()) nsIt->second->setSlaveMode(rdxCtx); @@ -426,7 +427,7 @@ Error ReindexerImpl::closeNamespace(std::string_view nsName, const RdxContext& c Namespace::Ptr ns; Error err; try { - ULock lock(mtx_, &ctx); + ULock lock(mtx_, ctx); auto nsIt = namespaces_.find(nsName); if (nsIt == namespaces_.end()) { return Error(errNotFound, "Namespace '%s' does not exist", nsName); @@ -498,7 +499,7 @@ Error ReindexerImpl::RenameNamespace(std::string_view srcNsName, const std::stri const auto rdxCtx = ctx.CreateRdxContext( ctx.NeedTraceActivity() ? (ser << "RENAME " << srcNsName << " to " << dstNsName).Slice() : ""sv, activities_); { - SLock lock(mtx_, &rdxCtx); + SLock lock(mtx_, rdxCtx); auto srcIt = namespaces_.find(srcNsName); if (srcIt == namespaces_.end()) { return Error(errParams, "Namespace '%s' doesn't exist", srcNsName); @@ -540,7 +541,7 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri { // Perform namespace flushes to minimize chances of the flush under lock - SLock lock(mtx_, &rdxCtx); + SLock lock(mtx_, rdxCtx); auto srcIt = namespaces_.find(srcNsName); srcNs = (srcIt != namespaces_.end()) ? srcIt->second : Namespace::Ptr(); lock.unlock(); @@ -554,7 +555,7 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri } } - ULock lock(mtx_, &rdxCtx); + ULock lock(mtx_, rdxCtx); auto srcIt = namespaces_.find(srcNsName); if (srcIt == namespaces_.end()) { return Error(errParams, "Namespace '%s' doesn't exist", srcNsName); @@ -837,7 +838,7 @@ Error ReindexerImpl::Select(std::string_view query, QueryResults& result, const } struct ItemRefLess { - bool operator()(const ItemRef& lhs, const ItemRef& rhs) const { + bool operator()(const ItemRef& lhs, const ItemRef& rhs) const noexcept { if (lhs.Proc() == rhs.Proc()) { if (lhs.Nsid() == rhs.Nsid()) { return lhs.Id() < rhs.Id(); @@ -1200,7 +1201,7 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& result.Erase(itemRefVec.begin(), itemRefVec.end()); return; } - std::sort(itemRefVec.begin(), itemRefVec.end(), ItemRefLess()); + boost::sort::pdqsort(itemRefVec.begin(), itemRefVec.end(), ItemRefLess()); if (q.HasOffset()) { result.Erase(itemRefVec.begin(), itemRefVec.begin() + q.Offset()); } @@ -1224,7 +1225,7 @@ Error ReindexerImpl::Commit(std::string_view /*_namespace*/) { } Namespace::Ptr ReindexerImpl::getNamespace(std::string_view nsName, const RdxContext& ctx) { - SLock lock(mtx_, &ctx); + SLock lock(mtx_, ctx); auto nsIt = namespaces_.find(nsName); if (nsIt == namespaces_.end()) { throw Error(errParams, "Namespace '%s' does not exist", nsName); @@ -1235,7 +1236,7 @@ Namespace::Ptr ReindexerImpl::getNamespace(std::string_view nsName, const RdxCon } Namespace::Ptr ReindexerImpl::getNamespaceNoThrow(std::string_view nsName, const RdxContext& ctx) { - SLock lock(mtx_, &ctx); + SLock lock(mtx_, ctx); auto nsIt = namespaces_.find(nsName); return (nsIt == namespaces_.end()) ? nullptr : nsIt->second; } @@ -1279,7 +1280,7 @@ Error ReindexerImpl::DropIndex(std::string_view nsName, const IndexDef& indexDef } std::vector> ReindexerImpl::getNamespaces(const RdxContext& ctx) { - SLock lock(mtx_, &ctx); + SLock lock(mtx_, ctx); std::vector> ret; ret.reserve(namespaces_.size()); for (auto& ns : namespaces_) { @@ -1290,7 +1291,7 @@ std::vector> ReindexerImpl::getNamespaces std::vector ReindexerImpl::getNamespacesNames(const RdxContext& ctx) { std::vector ret; - SLock lock(mtx_, &ctx); + SLock lock(mtx_, ctx); ret.reserve(namespaces_.size()); for (auto& ns : namespaces_) { ret.emplace_back(); @@ -1322,20 +1323,22 @@ Error ReindexerImpl::EnumNamespaces(std::vector& defs, EnumNamespa for (auto& d : dirs) { if (d.isDir && d.name != "." && d.name != ".." && opts.MatchFilter(d.name)) { { - SLock lock(mtx_, &rdxCtx); + SLock lock(mtx_, rdxCtx); if (namespaces_.find(d.name) != namespaces_.end()) continue; } std::unique_ptr tmpNs{new NamespaceImpl(d.name, observers_)}; try { tmpNs->EnableStorage(storagePath_, StorageOpts(), storageType_, rdxCtx); defs.push_back(tmpNs->GetDefinition(rdxCtx)); - } catch (reindexer::Error) { + // NOLINTBEGIN(bugprone-empty-catch) + } catch (const Error&) { } + // NOLINTEND(bugprone-empty-catch) } } } - } catch (reindexer::Error err) { - return err.code(); + } catch (const Error& err) { + return err; } return errOK; } @@ -1610,10 +1613,7 @@ void ReindexerImpl::updateReplicationConfFile() { auto err = replConfigFileChecker_.RewriteFile(std::string(ser.Slice()), [&oldReplConf](const std::string& content) { ReplicationConfigData replConf; Error err = replConf.FromYML(content); - if (err.ok()) { - return replConf == oldReplConf; - } - return false; + return err.ok() && (replConf == oldReplConf); }); if (!err.ok()) { throw err; @@ -1634,7 +1634,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) if (op == OpOr) { return std::nullopt; } - if (entries.HoldsOrReferTo(i)) { + if (entries.Is(i)) { auto& qe = entries.Get(i); if (qe.FieldName() == kNsNameField) { if (op == OpNot) { @@ -1659,7 +1659,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) res->emplace_back(v.As()); } } - } else if (entries.HoldsOrReferTo(i)) { + } else if (entries.Is(i)) { auto& qe = entries.Get(i); if (qe.LeftFieldName() == kNsNameField || qe.RightFieldName() == kNsNameField) { return std::nullopt; diff --git a/cpp_src/core/reindexerimpl.h b/cpp_src/core/reindexerimpl.h index f8ef5736d..d5ff74fbd 100644 --- a/cpp_src/core/reindexerimpl.h +++ b/cpp_src/core/reindexerimpl.h @@ -31,7 +31,7 @@ class ReindexerImpl { using Mutex = MarkedMutex; using StatsSelectMutex = MarkedMutex; struct NsLockerItem { - NsLockerItem(NamespaceImpl::Ptr ins = {}) : ns(std::move(ins)), count(1) {} + NsLockerItem(NamespaceImpl::Ptr ins = {}) noexcept : ns(std::move(ins)), count(1) {} NamespaceImpl::Ptr ns; NamespaceImpl::Locker::RLockT nsLck; unsigned count = 1; @@ -144,8 +144,9 @@ class ReindexerImpl { assertrx(0); } void Lock() { - std::sort(begin(), end(), [](const NsLockerItem &lhs, const NsLockerItem &rhs) { return lhs.ns.get() < rhs.ns.get(); }); - for (auto it = begin(); it != end(); ++it) { + boost::sort::pdqsort_branchless( + begin(), end(), [](const NsLockerItem &lhs, const NsLockerItem &rhs) noexcept { return lhs.ns.get() < rhs.ns.get(); }); + for (auto it = begin(), e = end(); it != e; ++it) { it->nsLck = it->ns->rLock(context_); } locked_ = true; diff --git a/cpp_src/core/selectfunc/nsselectfuncinterface.cc b/cpp_src/core/selectfunc/nsselectfuncinterface.cc index ba5c0c8ea..e5e470865 100644 --- a/cpp_src/core/selectfunc/nsselectfuncinterface.cc +++ b/cpp_src/core/selectfunc/nsselectfuncinterface.cc @@ -5,14 +5,14 @@ namespace reindexer { const std::string& NsSelectFuncInterface::GetName() const noexcept { return nm_.name_; }; int NsSelectFuncInterface::getIndexByName(std::string_view index) const noexcept { return nm_.getIndexByName(index); } -bool NsSelectFuncInterface::getIndexByName(std::string_view name, int& index) const noexcept { return nm_.getIndexByName(name, index); } +bool NsSelectFuncInterface::getIndexByName(std::string_view name, int& index) const noexcept { return nm_.tryGetIndexByName(name, index); } int NsSelectFuncInterface::getIndexesCount() const noexcept { return nm_.indexes_.size(); } const std::string& NsSelectFuncInterface::getIndexName(int id) const noexcept { return nm_.indexes_[id]->Name(); } IndexType NsSelectFuncInterface::getIndexType(int id) const noexcept { return nm_.indexes_[id]->Type(); } const FieldsSet& NsSelectFuncInterface::getIndexFields(int id) const noexcept { return nm_.indexes_[id]->Fields(); } TagsPath NsSelectFuncInterface::getTagsPathForField(std::string_view jsonPath) const noexcept { - return nm_.tagsMatcher_.path2tag(jsonPath); + return nm_.tagsMatcher_.path2tag(jsonPath); } } // namespace reindexer diff --git a/cpp_src/core/selectfunc/selectfunc.cc b/cpp_src/core/selectfunc/selectfunc.cc index 604571816..58ae7c734 100644 --- a/cpp_src/core/selectfunc/selectfunc.cc +++ b/cpp_src/core/selectfunc/selectfunc.cc @@ -38,7 +38,7 @@ SelectFunction::Ptr SelectFunctionsHolder::AddNamespace(const Query &q, const Na return queries_[nsid]; } -SelectFunction::SelectFunction(const Query &q, NsSelectFuncInterface &&nm) : nm_(std::move(nm)), currCjsonFieldIdx_(nm.getIndexesCount()) { +SelectFunction::SelectFunction(const Query &q, NsSelectFuncInterface &&nm) : nm_(std::move(nm)), currCjsonFieldIdx_(nm_.getIndexesCount()) { functions_.reserve(q.selectFunctions_.size()); for (auto &func : q.selectFunctions_) { SelectFuncParser parser; diff --git a/cpp_src/core/selectkeyresult.h b/cpp_src/core/selectkeyresult.h index 17a597eb0..55f83498b 100644 --- a/cpp_src/core/selectkeyresult.h +++ b/cpp_src/core/selectkeyresult.h @@ -199,18 +199,20 @@ class SelectKeyResult : public h_vector { size_t actualSize = 0; ids.resize(idsCount); auto rit = ids.begin(); - for (auto it = begin(); it != end(); ++it) { + for (auto it = begin(), endIt = end(); it != endIt; ++it) { if (it->isRange_) { throw Error(errLogic, "Unable to merge 'range' idset ('generic sort mode')"); } if (it->useBtree_) { - actualSize += it->set_->size(); + const auto sz = it->set_->size(); + actualSize += sz; std::copy(it->set_->begin(), it->set_->end(), rit); - rit += it->set_->size(); + rit += sz; } else { - actualSize += it->ids_.size(); + const auto sz = it->ids_.size(); + actualSize += sz; std::copy(it->ids_.begin(), it->ids_.end(), rit); - rit += it->ids_.size(); + rit += sz; } } assertrx(idsCount == actualSize); @@ -218,7 +220,7 @@ class SelectKeyResult : public h_vector { } else { mergedIds = make_intrusive>(); mergedIds->reserve(idsCount); - for (auto it = begin(); it != end(); ++it) { + for (auto it = begin(), endIt = end(); it != endIt; ++it) { if (it->isRange_) { throw Error(errLogic, "Unable to merge 'range' idset ('merge sort mode')"); } @@ -231,7 +233,7 @@ class SelectKeyResult : public h_vector { for (;;) { const int min = mergedIds->size() ? mergedIds->back() : INT_MIN; int curMin = INT_MAX; - for (auto it = begin(); it != end(); it++) { + for (auto it = begin(), endIt = end(); it != endIt; ++it) { if (it->useBtree_) { const auto end = it->set_->end(); for (; it->itset_ != end && *it->itset_ <= min; ++it->itset_) { @@ -250,8 +252,8 @@ class SelectKeyResult : public h_vector { mergedIds->shrink_to_fit(); } clear(); - emplace_back(SingleSelectKeyResult(mergedIds)); deferedExplicitSort = false; + emplace_back(mergedIds); return mergedIds; } }; diff --git a/cpp_src/core/transaction.cc b/cpp_src/core/transaction.cc index fc39bad7f..83f881da4 100644 --- a/cpp_src/core/transaction.cc +++ b/cpp_src/core/transaction.cc @@ -40,6 +40,14 @@ void Transaction::Modify(Query &&query) { if (impl_) impl_->Modify(std::move(query)); } +void Transaction::PutMeta(std::string_view key, std::string_view value) { + if (impl_) impl_->PutMeta(key, value); +} + +void Transaction::SetTagsMatcher(TagsMatcher &&tm) { + if (impl_) impl_->SetTagsMatcher(std::move(tm)); +} + Item Transaction::NewItem() { return impl_->NewItem(); } std::vector &Transaction::GetSteps() { diff --git a/cpp_src/core/transaction.h b/cpp_src/core/transaction.h index 0ab5c8b43..27709df4f 100644 --- a/cpp_src/core/transaction.h +++ b/cpp_src/core/transaction.h @@ -29,6 +29,8 @@ class Transaction { void Delete(Item &&item); void Modify(Item &&item, ItemModifyMode mode); void Modify(Query &&query); + void PutMeta(std::string_view key, std::string_view value); + void SetTagsMatcher(TagsMatcher &&tm); bool IsFree() { return impl_ == nullptr; } Item NewItem(); Item GetItem(TransactionStep &&st); diff --git a/cpp_src/core/transactionimpl.cc b/cpp_src/core/transactionimpl.cc index a7ccb4f16..e5c8e8853 100644 --- a/cpp_src/core/transactionimpl.cc +++ b/cpp_src/core/transactionimpl.cc @@ -7,18 +7,41 @@ namespace reindexer { void TransactionImpl::checkTagsMatcher(Item &item) { if (item.IsTagsUpdated()) { ItemImpl *ritem = item.impl_; - UpdateTagsMatcherFromItem(ritem); + if (ritem->Type().get() != payloadType_.get() || !tagsMatcher_.try_merge(ritem->tagsMatcher())) { + std::string jsonSliceBuf(ritem->GetJSON()); + + ItemImpl tmpItem(payloadType_, tagsMatcher_); + tmpItem.Value().SetLSN(ritem->Value().GetLSN()); + *ritem = std::move(tmpItem); + + auto err = ritem->FromJSON(jsonSliceBuf, nullptr); + if (!err.ok()) throw err; + + if (ritem->tagsMatcher().isUpdated() && !tagsMatcher_.try_merge(ritem->tagsMatcher())) + throw Error(errLogic, "Could not insert item. TagsMatcher was not merged."); + ritem->tagsMatcher() = tagsMatcher_; + ritem->tagsMatcher().setUpdated(); + } else { + ritem->tagsMatcher() = tagsMatcher_; + ritem->tagsMatcher().setUpdated(); + } tagsUpdated_ = true; } } Item TransactionImpl::NewItem() { std::unique_lock lock(mtx_); - return Item(new ItemImpl(payloadType_, tagsMatcher_, pkFields_)); + Item item(new ItemImpl(payloadType_, tagsMatcher_, pkFields_)); + item.impl_->tagsMatcher().clearUpdated(); + return item; } + Item TransactionImpl::GetItem(TransactionStep &&st) { std::unique_lock lock(mtx_); - return Item(new ItemImpl(payloadType_, tagsMatcher_, pkFields_, schema_, std::move(st.itemData_))); + auto &data = std::get(st.data_); + auto item = Item(new ItemImpl(payloadType_, tagsMatcher_, pkFields_, schema_, std::move(data.data))); + data.hadTmUpdate ? item.impl_->tagsMatcher().setUpdated() : item.impl_->tagsMatcher().clearUpdated(); + return item; } void TransactionImpl::ValidatePK(const FieldsSet &pkFields) { @@ -30,58 +53,57 @@ void TransactionImpl::ValidatePK(const FieldsSet &pkFields) { } } -void TransactionImpl::UpdateTagsMatcherFromItem(ItemImpl *ritem) { - if (ritem->Type().get() != payloadType_.get() || (ritem->tagsMatcher().isUpdated() && !tagsMatcher_.try_merge(ritem->tagsMatcher()))) { - std::string jsonSliceBuf(ritem->GetJSON()); - - ItemImpl tmpItem(payloadType_, tagsMatcher_); - tmpItem.Value().SetLSN(ritem->Value().GetLSN()); - *ritem = std::move(tmpItem); - - auto err = ritem->FromJSON(jsonSliceBuf, nullptr); - if (!err.ok()) throw err; - - if (ritem->tagsMatcher().isUpdated() && !tagsMatcher_.try_merge(ritem->tagsMatcher())) - throw Error(errLogic, "Could not insert item. TagsMatcher was not merged."); - ritem->tagsMatcher() = tagsMatcher_; - ritem->tagsMatcher().setUpdated(); - } - if (ritem->tagsMatcher().isUpdated()) { - ritem->tagsMatcher() = tagsMatcher_; - ritem->tagsMatcher().setUpdated(); - } -} - void TransactionImpl::Insert(Item &&item) { std::unique_lock lock(mtx_); checkTagsMatcher(item); - steps_.emplace_back(TransactionStep{std::move(item), ModeInsert}); + steps_.emplace_back(std::move(item), ModeInsert); } void TransactionImpl::Update(Item &&item) { std::unique_lock lock(mtx_); checkTagsMatcher(item); - steps_.emplace_back(TransactionStep{std::move(item), ModeUpdate}); + steps_.emplace_back(std::move(item), ModeUpdate); } void TransactionImpl::Upsert(Item &&item) { std::unique_lock lock(mtx_); checkTagsMatcher(item); - steps_.emplace_back(TransactionStep{std::move(item), ModeUpsert}); + steps_.emplace_back(std::move(item), ModeUpsert); } void TransactionImpl::Delete(Item &&item) { std::unique_lock lock(mtx_); checkTagsMatcher(item); - steps_.emplace_back(TransactionStep{std::move(item), ModeDelete}); + steps_.emplace_back(std::move(item), ModeDelete); } void TransactionImpl::Modify(Item &&item, ItemModifyMode mode) { std::unique_lock lock(mtx_); checkTagsMatcher(item); hasDeleteItemSteps_ = hasDeleteItemSteps_ || (mode == ModeDelete); - steps_.emplace_back(TransactionStep{std::move(item), mode}); + steps_.emplace_back(std::move(item), mode); } void TransactionImpl::Modify(Query &&query) { std::unique_lock lock(mtx_); - steps_.emplace_back(TransactionStep(std::move(query))); + steps_.emplace_back(std::move(query)); +} + +void TransactionImpl::PutMeta(std::string_view key, std::string_view value) { + if (key.empty()) { + throw Error(errLogic, "Empty meta key is not allowed in tx"); + } + + std::lock_guard lock(mtx_); + steps_.emplace_back(key, value); +} + +void TransactionImpl::SetTagsMatcher(TagsMatcher &&tm) { + std::lock_guard lock(mtx_); + // NOTE: In v4 tm tokens here are always the same, but in v3 those tokens are not synchronized. Probably it should workd anyway + // if (tm.stateToken() != tagsMatcher_.stateToken()) { + // throw Error(errParams, "Tx tm statetoken missmatch: %08X vs %08X", tagsMatcher_.stateToken(), tm.stateToken()); + // } + tagsMatcher_ = tm; + tagsMatcher_.UpdatePayloadType(payloadType_, false); + tagsUpdated_ = true; + steps_.emplace_back(std::move(tm)); } } // namespace reindexer diff --git a/cpp_src/core/transactionimpl.h b/cpp_src/core/transactionimpl.h index 89baa10d9..36403c28e 100644 --- a/cpp_src/core/transactionimpl.h +++ b/cpp_src/core/transactionimpl.h @@ -5,22 +5,46 @@ namespace reindexer { +struct TransactionItemStep { + ItemModifyMode mode; + bool hadTmUpdate; + ItemImplRawData data; +}; + +struct TransactionQueryStep { + std::unique_ptr query; +}; + +struct TransactionMetaStep { + std::string key; + std::string value; +}; + +struct TransactionTmStep { + TagsMatcher tm; +}; + class TransactionStep { public: - TransactionStep(Item &&item, ItemModifyMode modifyMode) : itemData_(std::move(*item.impl_)), modifyMode_(modifyMode), query_(nullptr) { + enum class Type : uint8_t { Nop, ModifyItem, Query, PutMeta, SetTM }; + + TransactionStep(Item &&item, ItemModifyMode modifyMode) + : data_(TransactionItemStep{modifyMode, item.IsTagsUpdated(), std::move(*item.impl_)}), type_(Type::ModifyItem) { delete item.impl_; item.impl_ = nullptr; } - TransactionStep(Query &&query) : modifyMode_(ModeUpdate), query_(new Query(std::move(query))) {} + TransactionStep(TagsMatcher &&tm) : data_(TransactionTmStep{std::move(tm)}), type_(Type::SetTM) {} + TransactionStep(Query &&query) : data_(TransactionQueryStep{std::make_unique(std::move(query))}), type_(Type::Query) {} + TransactionStep(std::string_view key, std::string_view value) + : data_(TransactionMetaStep{std::string(key), std::string(value)}), type_(Type::PutMeta) {} TransactionStep(const TransactionStep &) = delete; TransactionStep &operator=(const TransactionStep &) = delete; - TransactionStep(TransactionStep && /*rhs*/) noexcept = default; - TransactionStep &operator=(TransactionStep && /*rhs*/) = default; + TransactionStep(TransactionStep && /*rhs*/) = default; + TransactionStep &operator=(TransactionStep && /*rhs*/) = delete; - ItemImplRawData itemData_; - ItemModifyMode modifyMode_; - std::unique_ptr query_; + std::variant data_; + Type type_; }; class TransactionImpl { @@ -34,7 +58,9 @@ class TransactionImpl { nsName_(nsName), tagsUpdated_(false), hasDeleteItemSteps_(false), - startTime_(std::chrono::high_resolution_clock::now()) {} + startTime_(std::chrono::high_resolution_clock::now()) { + tagsMatcher_.clearUpdated(); + } void Insert(Item &&item); void Update(Item &&item); @@ -42,8 +68,9 @@ class TransactionImpl { void Delete(Item &&item); void Modify(Item &&item, ItemModifyMode mode); void Modify(Query &&item); + void PutMeta(std::string_view key, std::string_view value); + void SetTagsMatcher(TagsMatcher &&tm); - void UpdateTagsMatcherFromItem(ItemImpl *ritem); Item NewItem(); Item GetItem(TransactionStep &&st); void ValidatePK(const FieldsSet &pkFields); diff --git a/cpp_src/debug/terminate_handler.cpp b/cpp_src/debug/terminate_handler.cpp index 7269ccde5..b2a365d56 100644 --- a/cpp_src/debug/terminate_handler.cpp +++ b/cpp_src/debug/terminate_handler.cpp @@ -30,6 +30,7 @@ static void terminate_handler() { } catch (Error &err) { sout << ": " << err.what(); } catch (...) { + sout << ": "; } sout << " ***" << std::endl; } else { diff --git a/cpp_src/estl/contexted_locks.h b/cpp_src/estl/contexted_locks.h index ba8dc9045..2e60b0ec8 100644 --- a/cpp_src/estl/contexted_locks.h +++ b/cpp_src/estl/contexted_locks.h @@ -13,32 +13,27 @@ using std::adopt_lock_t; namespace reindexer { -const milliseconds kDefaultCondChkTime = milliseconds(20); +constexpr milliseconds kDefaultCondChkTime = milliseconds(20); template class contexted_unique_lock { public: using MutexType = _Mutex; - explicit contexted_unique_lock() : _M_mtx(nullptr), _M_owns(false), _M_context(nullptr), _M_chkTimeout(kDefaultCondChkTime) {} - explicit contexted_unique_lock(MutexType& __mtx, Context* __context, milliseconds __chkTimeout = kDefaultCondChkTime) - : _M_mtx(&__mtx), _M_owns(false), _M_context(__context), _M_chkTimeout(__chkTimeout) { - assertrx(_M_context); + explicit contexted_unique_lock() noexcept : _M_mtx(nullptr), _M_owns(false), _M_context(nullptr), _M_chkTimeout(kDefaultCondChkTime) {} + explicit contexted_unique_lock(MutexType& __mtx, Context& __context, milliseconds __chkTimeout = kDefaultCondChkTime) + : _M_mtx(&__mtx), _M_owns(false), _M_context(&__context), _M_chkTimeout(__chkTimeout) { lock(); } - explicit contexted_unique_lock(MutexType& __mtx, defer_lock_t, Context* __context, milliseconds __chkTimeout = kDefaultCondChkTime) - : _M_mtx(&__mtx), _M_owns(false), _M_context(__context), _M_chkTimeout(__chkTimeout) { - assertrx(_M_context); - } - explicit contexted_unique_lock(MutexType& __mtx, adopt_lock_t, Context* __context, milliseconds __chkTimeout = kDefaultCondChkTime) - : _M_mtx(&__mtx), _M_owns(true), _M_context(__context), _M_chkTimeout(__chkTimeout) { - assertrx(_M_context); - } - explicit contexted_unique_lock(MutexType& __mtx, try_to_lock_t, Context* __context, milliseconds __chkTimeout = kDefaultCondChkTime) - : _M_mtx(&__mtx), _M_owns(__mtx.try_lock()), _M_context(__context), _M_chkTimeout(__chkTimeout) { - assertrx(_M_context); - } - contexted_unique_lock(contexted_unique_lock&& lck) + explicit contexted_unique_lock(MutexType& __mtx, defer_lock_t, Context& __context, + milliseconds __chkTimeout = kDefaultCondChkTime) noexcept + : _M_mtx(&__mtx), _M_owns(false), _M_context(&__context), _M_chkTimeout(__chkTimeout) {} + explicit contexted_unique_lock(MutexType& __mtx, adopt_lock_t, Context& __context, + milliseconds __chkTimeout = kDefaultCondChkTime) noexcept + : _M_mtx(&__mtx), _M_owns(true), _M_context(&__context), _M_chkTimeout(__chkTimeout) {} + explicit contexted_unique_lock(MutexType& __mtx, try_to_lock_t, Context& __context, milliseconds __chkTimeout = kDefaultCondChkTime) + : _M_mtx(&__mtx), _M_owns(__mtx.try_lock()), _M_context(&__context), _M_chkTimeout(__chkTimeout) {} + contexted_unique_lock(contexted_unique_lock&& lck) noexcept : _M_mtx(lck._M_mtx), _M_owns(lck._M_owns), _M_context(lck._M_context), _M_chkTimeout(lck._M_chkTimeout) { lck._M_owns = false; lck._M_mtx = nullptr; @@ -50,7 +45,7 @@ class contexted_unique_lock { contexted_unique_lock(const contexted_unique_lock&) = delete; contexted_unique_lock& operator=(const contexted_unique_lock&) = delete; - contexted_unique_lock& operator=(contexted_unique_lock&& lck) { + contexted_unique_lock& operator=(contexted_unique_lock&& lck) noexcept { if (this != &lck) { if (_M_owns) unlock(); _M_mtx = lck._M_mtx; @@ -102,7 +97,7 @@ class contexted_unique_lock { MutexType* mutex() const noexcept { return _M_mtx; } private: - void _M_lockable() const { + void _M_lockable() const noexcept { if (_M_mtx == nullptr) assertrx(0); if (_M_owns) assertrx(0); } @@ -118,21 +113,17 @@ class contexted_shared_lock { public: using MutexType = _Mutex; - explicit contexted_shared_lock() : _M_mtx(nullptr), _M_owns(false), _M_context(nullptr), _M_chkTimeout(kDefaultCondChkTime) {} - explicit contexted_shared_lock(MutexType& __mtx, Context* __context, milliseconds __chkTimeout = kDefaultCondChkTime) - : _M_mtx(&__mtx), _M_owns(false), _M_context(__context), _M_chkTimeout(__chkTimeout) { - assertrx(_M_context); + explicit contexted_shared_lock() noexcept : _M_mtx(nullptr), _M_owns(false), _M_context(nullptr), _M_chkTimeout(kDefaultCondChkTime) {} + explicit contexted_shared_lock(MutexType& __mtx, Context& __context, milliseconds __chkTimeout = kDefaultCondChkTime) + : _M_mtx(&__mtx), _M_owns(false), _M_context(&__context), _M_chkTimeout(__chkTimeout) { lock(); } - explicit contexted_shared_lock(MutexType& __mtx, adopt_lock_t, Context* __context, milliseconds __chkTimeout = kDefaultCondChkTime) - : _M_mtx(&__mtx), _M_owns(true), _M_context(__context), _M_chkTimeout(__chkTimeout) { - assertrx(_M_context); - } - explicit contexted_shared_lock(MutexType& __mtx, try_to_lock_t, Context* __context, milliseconds __chkTimeout = kDefaultCondChkTime) - : _M_mtx(&__mtx), _M_owns(__mtx.try_lock()), _M_context(__context), _M_chkTimeout(__chkTimeout) { - assertrx(_M_context); - } - contexted_shared_lock(contexted_shared_lock&& lck) + explicit contexted_shared_lock(MutexType& __mtx, adopt_lock_t, Context& __context, + milliseconds __chkTimeout = kDefaultCondChkTime) noexcept + : _M_mtx(&__mtx), _M_owns(true), _M_context(&__context), _M_chkTimeout(__chkTimeout) {} + explicit contexted_shared_lock(MutexType& __mtx, try_to_lock_t, Context& __context, milliseconds __chkTimeout = kDefaultCondChkTime) + : _M_mtx(&__mtx), _M_owns(__mtx.try_lock()), _M_context(&__context), _M_chkTimeout(__chkTimeout) {} + contexted_shared_lock(contexted_shared_lock&& lck) noexcept : _M_mtx(lck._M_mtx), _M_owns(lck._M_owns), _M_context(lck._M_context), _M_chkTimeout(lck._M_chkTimeout) { lck._M_owns = false; lck._M_mtx = nullptr; @@ -144,7 +135,7 @@ class contexted_shared_lock { contexted_shared_lock(const contexted_shared_lock&) = delete; contexted_shared_lock& operator=(const contexted_shared_lock&) = delete; - contexted_shared_lock& operator=(contexted_shared_lock&& lck) { + contexted_shared_lock& operator=(contexted_shared_lock&& lck) noexcept { if (this != &lck) { if (_M_owns) unlock(); _M_mtx = lck._M_mtx; @@ -196,7 +187,7 @@ class contexted_shared_lock { MutexType* mutex() const noexcept { return _M_mtx; } private: - void _M_lockable() const { + void _M_lockable() const noexcept { if (_M_mtx == nullptr) assertrx(0); if (_M_owns) assertrx(0); } diff --git a/cpp_src/estl/h_vector.h b/cpp_src/estl/h_vector.h index 1f26945d3..735ce3f55 100644 --- a/cpp_src/estl/h_vector.h +++ b/cpp_src/estl/h_vector.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "debug_macros.h" #include "trivial_reverse_iterator.h" @@ -123,7 +124,7 @@ class h_vector { return *this; } - bool operator==(const h_vector& other) const noexcept { + bool operator==(const h_vector& other) const noexcept(noexcept(std::declval() == std::declval())) { if (&other != this) { if (size() != other.size()) return false; for (size_t i = 0; i < size(); ++i) { @@ -133,7 +134,9 @@ class h_vector { } return true; } - bool operator!=(const h_vector& other) const noexcept { return !operator==(other); } + bool operator!=(const h_vector& other) const noexcept(noexcept(std::declval() == std::declval())) { + return !operator==(other); + } template void clear() noexcept { diff --git a/cpp_src/gtests/bench/fixtures/api_tv_simple.cc b/cpp_src/gtests/bench/fixtures/api_tv_simple.cc index e628e2b5f..bdf2cb7a5 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_simple.cc +++ b/cpp_src/gtests/bench/fixtures/api_tv_simple.cc @@ -81,9 +81,9 @@ void ApiTvSimple::RegisterAllCases() { Register("Query2CondLeftJoin", &ApiTvSimple::Query2CondLeftJoin, this); Register("Query2CondLeftJoinTotal", &ApiTvSimple::Query2CondLeftJoinTotal, this); Register("Query2CondLeftJoinCachedTotal", &ApiTvSimple::Query2CondLeftJoinCachedTotal, this); - Register("Query0CondInnerJoinUnlimit", &ApiTvSimple::Query0CondInnerJoinUnlimit, this); - Register("Query0CondInnerJoinUnlimitLowSelectivity", &ApiTvSimple::Query0CondInnerJoinUnlimitLowSelectivity, this); - Register("Query0CondInnerJoinPreResultStoreValues", &ApiTvSimple::Query0CondInnerJoinPreResultStoreValues, this); + Register("Query0CondInnerJoinUnlimit", &ApiTvSimple::Query0CondInnerJoinUnlimit, this)->Iterations(500); + Register("Query0CondInnerJoinUnlimitLowSelectivity", &ApiTvSimple::Query0CondInnerJoinUnlimitLowSelectivity, this)->Iterations(500); + Register("Query0CondInnerJoinPreResultStoreValues", &ApiTvSimple::Query0CondInnerJoinPreResultStoreValues, this)->Iterations(500); Register("Query2CondInnerJoin", &ApiTvSimple::Query2CondInnerJoin, this); Register("Query2CondInnerJoinTotal", &ApiTvSimple::Query2CondInnerJoinTotal, this); Register("Query2CondInnerJoinCachedTotal", &ApiTvSimple::Query2CondInnerJoinCachedTotal, this); @@ -95,9 +95,9 @@ void ApiTvSimple::RegisterAllCases() { Register("Query4Cond", &ApiTvSimple::Query4Cond, this); Register("Query4CondTotal", &ApiTvSimple::Query4CondTotal, this); Register("Query4CondCachedTotal", &ApiTvSimple::Query4CondCachedTotal, this); - Register("Query4CondRange", &ApiTvSimple::Query4CondRange, this); - Register("Query4CondRangeTotal", &ApiTvSimple::Query4CondRangeTotal, this); - Register("Query4CondRangeCachedTotal", &ApiTvSimple::Query4CondRangeCachedTotal, this); + Register("Query4CondRange", &ApiTvSimple::Query4CondRange, this)->Iterations(1000); + Register("Query4CondRangeTotal", &ApiTvSimple::Query4CondRangeTotal, this)->Iterations(1000); + Register("Query4CondRangeCachedTotal", &ApiTvSimple::Query4CondRangeCachedTotal, this)->Iterations(1000); #if !defined(REINDEX_WITH_ASAN) && !defined(REINDEX_WITH_TSAN) && !defined(RX_WITH_STDLIB_DEBUG) Register("Query2CondIdSet10", &ApiTvSimple::Query2CondIdSet10, this); #endif // !defined(REINDEX_WITH_ASAN) && !defined(REINDEX_WITH_TSAN) @@ -111,7 +111,7 @@ void ApiTvSimple::RegisterAllCases() { Register("FromCJSONPKOnly", &ApiTvSimple::FromCJSONPKOnly, this); Register("GetCJSON", &ApiTvSimple::GetCJSON, this); Register("ExtractField", &ApiTvSimple::ExtractField, this); - // NOLINTEND(*cplusplus.NewDeleteLeaks) + // NOLINTEND(*cplusplus.NewDeleteLeaks) } reindexer::Error ApiTvSimple::Initialize() { @@ -1064,7 +1064,7 @@ void ApiTvSimple::Query4CondCachedTotal(benchmark::State& state) { void ApiTvSimple::Query4CondRange(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) - int startTime = random(0, 50000); + int startTime = random(0, 30000); int endTime = startTime + 10000; Query q(nsdef_.name); q.Limit(20) @@ -1083,7 +1083,7 @@ void ApiTvSimple::Query4CondRange(benchmark::State& state) { void ApiTvSimple::Query4CondRangeTotal(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) - int startTime = random(0, 50000); + int startTime = random(0, 30000); int endTime = startTime + 10000; Query q(nsdef_.name); q.Limit(20) @@ -1103,7 +1103,7 @@ void ApiTvSimple::Query4CondRangeTotal(benchmark::State& state) { void ApiTvSimple::Query4CondRangeCachedTotal(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) - int startTime = random(0, 50000); + int startTime = random(0, 30000); int endTime = startTime + 10000; Query q(nsdef_.name); q.Limit(20) diff --git a/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc b/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc index 48a1306f0..30846061c 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc +++ b/cpp_src/gtests/bench/fixtures/api_tv_simple_comparators.cc @@ -454,7 +454,7 @@ void ApiTvSimpleComparators::Query4CondCachedTotal(benchmark::State& state) { void ApiTvSimpleComparators::Query4CondRange(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) - int startTime = random(0, 50000); + int startTime = random(0, 30000); int endTime = startTime + 10000; Query q(nsdef_.name); q.Limit(20) @@ -473,7 +473,7 @@ void ApiTvSimpleComparators::Query4CondRange(benchmark::State& state) { void ApiTvSimpleComparators::Query4CondRangeTotal(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) - int startTime = random(0, 50000); + int startTime = random(0, 30000); int endTime = startTime + 10000; Query q(nsdef_.name); q.Limit(20) @@ -493,7 +493,7 @@ void ApiTvSimpleComparators::Query4CondRangeTotal(benchmark::State& state) { void ApiTvSimpleComparators::Query4CondRangeCachedTotal(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) - int startTime = random(0, 50000); + int startTime = random(0, 30000); int endTime = startTime + 10000; Query q(nsdef_.name); q.Limit(20) diff --git a/cpp_src/gtests/bench/fixtures/ft_fixture.cc b/cpp_src/gtests/bench/fixtures/ft_fixture.cc index 955ee9d66..c504e6a49 100644 --- a/cpp_src/gtests/bench/fixtures/ft_fixture.cc +++ b/cpp_src/gtests/bench/fixtures/ft_fixture.cc @@ -112,51 +112,50 @@ reindexer::Error FullText::Initialize() { return {}; } -void FullText::RegisterAllCases(size_t iterationCount) { +void FullText::RegisterAllCases(std::optional fastIterationCount, std::optional slowIterationCount) { constexpr static auto Mem = reindexer::FtFastConfig::Optimization::Memory; constexpr static auto CPU = reindexer::FtFastConfig::Optimization::CPU; - RegisterWrapper wrap(iterationCount); //-1 test limit - default time + RegisterWrapper wrapSlow(slowIterationCount); // std::numeric_limits::max() test limit - default time + RegisterWrapper wrapFast(fastIterationCount); // NOLINTBEGIN(*cplusplus.NewDeleteLeaks) Register("BuildAndInsertNs2", &FullText::BuildAndInsertLowWordsDiversityNs, this)->Iterations(1000)->Unit(benchmark::kMicrosecond); - wrap.SetOptions(Register("Fast3PhraseLowDiversity", &FullText::Fast3PhraseLowDiversity, this)); - wrap.SetOptions(Register("Fast3WordsLowDiversity", &FullText::Fast3WordsLowDiversity, this)); + wrapSlow.SetOptions(Register("Fast3PhraseLowDiversity", &FullText::Fast3PhraseLowDiversity, this)); + wrapSlow.SetOptions(Register("Fast3WordsLowDiversity", &FullText::Fast3WordsLowDiversity, this)); - wrap.SetOptions(Register("Fast3WordsWithAreasLowDiversity", &FullText::Fast3WordsWithAreasLowDiversity, this)); - wrap.SetOptions(Register("Fast3PhraseWithAreasLowDiversity", &FullText::Fast3PhraseWithAreasLowDiversity, this)); + wrapSlow.SetOptions(Register("Fast3WordsWithAreasLowDiversity", &FullText::Fast3WordsWithAreasLowDiversity, this)); + wrapSlow.SetOptions(Register("Fast3PhraseWithAreasLowDiversity", &FullText::Fast3PhraseWithAreasLowDiversity, this)); - wrap.SetOptions(Register("Fast2PhraseLowDiversity", &FullText::Fast2PhraseLowDiversity, this)); - wrap.SetOptions(Register("Fast2AndWordLowDiversity", &FullText::Fast2AndWordLowDiversity, this)); + wrapFast.SetOptions(Register("Fast2PhraseLowDiversity", &FullText::Fast2PhraseLowDiversity, this)); + wrapFast.SetOptions(Register("Fast2AndWordLowDiversity", &FullText::Fast2AndWordLowDiversity, this)); Register("Insert", &FullText::Insert, this)->Iterations(id_seq_->Count())->Unit(benchmark::kMicrosecond); - // Register("BuildCommonIndexes", &FullText::BuildCommonIndexes, this)->Iterations(1)->Unit(benchmark::kMicrosecond); Register("BuildFastTextIndex", &FullText::BuildFastTextIndex, this)->Iterations(1)->Unit(benchmark::kMicrosecond); - // Register("BuildFuzzyTextIndex", &FullText::BuildFuzzyTextIndex, this)->Iterations(1)->Unit(benchmark::kMicrosecond); - - wrap.SetOptions(Register("Fast1WordMatch.OptByMem", &FullText::Fast1WordMatch, this)); - wrap.SetOptions(Register("Fast2WordsMatch.OptByMem", &FullText::Fast2WordsMatch, this)); - wrap.SetOptions(Register("Fast1PrefixMatch.OptByMem", &FullText::Fast1PrefixMatch, this)); - wrap.SetOptions(Register("Fast2PrefixMatch.OptByMem", &FullText::Fast2PrefixMatch, this)); - wrap.SetOptions(Register("Fast1SuffixMatch.OptByMem", &FullText::Fast1SuffixMatch, this)); - wrap.SetOptions(Register("Fast2SuffixMatch.OptByMem", &FullText::Fast2SuffixMatch, this)); - - wrap.SetOptions(Register("Fast1TypoWordMatch.OptByMem", &FullText::Fast1TypoWordMatch, this)); - wrap.SetOptions(Register("Fast2TypoWordMatch.OptByMem", &FullText::Fast2TypoWordMatch, this)); - wrap.SetOptions(Register("Fast1WordWithAreaHighDiversity.optByMem", &FullText::Fast1WordWithAreaHighDiversity, this)); + wrapFast.SetOptions(Register("Fast1WordMatch.OptByMem", &FullText::Fast1WordMatch, this)); + wrapFast.SetOptions(Register("Fast2WordsMatch.OptByMem", &FullText::Fast2WordsMatch, this)); + wrapSlow.SetOptions(Register("Fast1PrefixMatch.OptByMem", &FullText::Fast1PrefixMatch, this)); + wrapSlow.SetOptions(Register("Fast2PrefixMatch.OptByMem", &FullText::Fast2PrefixMatch, this)); + wrapSlow.SetOptions(Register("Fast1SuffixMatch.OptByMem", &FullText::Fast1SuffixMatch, this)); + wrapSlow.SetOptions(Register("Fast2SuffixMatch.OptByMem", &FullText::Fast2SuffixMatch, this)); + wrapFast.SetOptions(Register("Fast1TypoWordMatch.OptByMem", &FullText::Fast1TypoWordMatch, this)); + wrapFast.SetOptions(Register("Fast2TypoWordMatch.OptByMem", &FullText::Fast2TypoWordMatch, this)); + wrapFast.SetOptions(Register("Fast1WordWithAreaHighDiversity.OptByMem", &FullText::Fast1WordWithAreaHighDiversity, this)); Register("SetOptimizationByCPU", &FullText::UpdateIndex, this)->Iterations(1)->Unit(benchmark::kMicrosecond); - wrap.SetOptions(Register("Fast1WordMatch.OptByCPU", &FullText::Fast1WordMatch, this)); + wrapFast.SetOptions(Register("Fast1WordMatch.OptByCPU", &FullText::Fast1WordMatch, this)); + wrapFast.SetOptions(Register("Fast2WordsMatch.OptByCPU", &FullText::Fast2WordsMatch, this)); + wrapSlow.SetOptions(Register("Fast1PrefixMatch.OptByCPU", &FullText::Fast1PrefixMatch, this)); + wrapSlow.SetOptions(Register("Fast2PrefixMatch.OptByCPU", &FullText::Fast2PrefixMatch, this)); + wrapSlow.SetOptions(Register("Fast1SuffixMatch.OptByCPU", &FullText::Fast1SuffixMatch, this)); + wrapSlow.SetOptions(Register("Fast2SuffixMatch.OptByCPU", &FullText::Fast2SuffixMatch, this)); + wrapFast.SetOptions(Register("Fast1TypoWordMatch.OptByCPU", &FullText::Fast1TypoWordMatch, this)); + wrapFast.SetOptions(Register("Fast2TypoWordMatch.OptByCPU", &FullText::Fast2TypoWordMatch, this)); + wrapFast.SetOptions(Register("Fast1WordWithAreaHighDiversity.OptByCPU", &FullText::Fast1WordWithAreaHighDiversity, this)); - wrap.SetOptions(Register("Fast2WordsMatch.OptByCPU", &FullText::Fast2WordsMatch, this)); - wrap.SetOptions(Register("Fast1PrefixMatch.OptByCPU", &FullText::Fast1PrefixMatch, this)); - wrap.SetOptions(Register("Fast2PrefixMatch.OptByCPU", &FullText::Fast2PrefixMatch, this)); - wrap.SetOptions(Register("Fast1SuffixMatch.OptByCPU", &FullText::Fast1SuffixMatch, this)); - wrap.SetOptions(Register("Fast2SuffixMatch.OptByCPU", &FullText::Fast2SuffixMatch, this)); - wrap.SetOptions(Register("Fast1TypoWordMatch.OptByCPU", &FullText::Fast1TypoWordMatch, this)); - wrap.SetOptions(Register("Fast2TypoWordMatch.OptByCPU", &FullText::Fast2TypoWordMatch, this)); + // Register("BuildFuzzyTextIndex", &FullText::BuildFuzzyTextIndex, this)->Iterations(1)->Unit(benchmark::kMicrosecond); // Register("Fuzzy1WordMatch", &FullText::Fuzzy1WordMatch, this)->Unit(benchmark::kMicrosecond); // Register("Fuzzy2WordsMatch", &FullText::Fuzzy2WordsMatch, this)->Unit(benchmark::kMicrosecond); @@ -198,17 +197,19 @@ void FullText::RegisterAllCases(size_t iterationCount) { Register("InitForAlternatingUpdatesAndSelects.OptByMem", &FullText::InitForAlternatingUpdatesAndSelects, this) ->Iterations(1) ->Unit(benchmark::kMicrosecond); - wrap.SetOptions(Register("AlternatingUpdatesAndSelects.OptByMem", &FullText::AlternatingUpdatesAndSelects, this)); - wrap.SetOptions(Register("AlternatingUpdatesAndSelectsByComposite.OptByMem", &FullText::AlternatingUpdatesAndSelectsByComposite, this)); - wrap.SetOptions(Register("AlternatingUpdatesAndSelectsByCompositeByNotIndexFields.OptByMem", - &FullText::AlternatingUpdatesAndSelectsByCompositeByNotIndexFields, this)); + wrapFast.SetOptions(Register("AlternatingUpdatesAndSelects.OptByMem", &FullText::AlternatingUpdatesAndSelects, this)); + wrapFast.SetOptions( + Register("AlternatingUpdatesAndSelectsByComposite.OptByMem", &FullText::AlternatingUpdatesAndSelectsByComposite, this)); + wrapFast.SetOptions(Register("AlternatingUpdatesAndSelectsByCompositeByNotIndexFields.OptByMem", + &FullText::AlternatingUpdatesAndSelectsByCompositeByNotIndexFields, this)); Register("InitForAlternatingUpdatesAndSelects.OptByCPU", &FullText::InitForAlternatingUpdatesAndSelects, this) ->Iterations(1) ->Unit(benchmark::kMicrosecond); - wrap.SetOptions(Register("AlternatingUpdatesAndSelects.OptByCPU", &FullText::AlternatingUpdatesAndSelects, this)); - wrap.SetOptions(Register("AlternatingUpdatesAndSelectsByComposite.OptByCPU", &FullText::AlternatingUpdatesAndSelectsByComposite, this)); - wrap.SetOptions(Register("AlternatingUpdatesAndSelectsByCompositeByNotIndexFields.OptByCPU", - &FullText::AlternatingUpdatesAndSelectsByCompositeByNotIndexFields, this)); + wrapFast.SetOptions(Register("AlternatingUpdatesAndSelects.OptByCPU", &FullText::AlternatingUpdatesAndSelects, this)); + wrapFast.SetOptions( + Register("AlternatingUpdatesAndSelectsByComposite.OptByCPU", &FullText::AlternatingUpdatesAndSelectsByComposite, this)); + wrapFast.SetOptions(Register("AlternatingUpdatesAndSelectsByCompositeByNotIndexFields.OptByCPU", + &FullText::AlternatingUpdatesAndSelectsByCompositeByNotIndexFields, this)); // NOLINTEND(*cplusplus.NewDeleteLeaks) } diff --git a/cpp_src/gtests/bench/fixtures/ft_fixture.h b/cpp_src/gtests/bench/fixtures/ft_fixture.h index 13f69eb43..954e89c7e 100644 --- a/cpp_src/gtests/bench/fixtures/ft_fixture.h +++ b/cpp_src/gtests/bench/fixtures/ft_fixture.h @@ -19,7 +19,7 @@ class FullText : private BaseFixture { FullText(Reindexer* db, const std::string& name, size_t maxItems); virtual reindexer::Error Initialize() override; - void RegisterAllCases(size_t iterationCount = -1); + void RegisterAllCases(std::optional fastIterationCount, std::optional slowIterationCount); private: virtual reindexer::Item MakeItem(benchmark::State&) override; @@ -101,18 +101,16 @@ class FullText : private BaseFixture { class RegisterWrapper { public: - //-1 test iteration limit - time - RegisterWrapper(size_t iterationCoun = -1) : iterationCoun_(iterationCoun) {} - Benchmark* SetOptions(Benchmark* b) { - b = b->Unit(benchmark::kMicrosecond); - if (iterationCoun_ != size_t(-1)) { - b = b->Iterations(iterationCoun_); + RegisterWrapper(std::optional iterationCoun) noexcept : iterationCoun_(iterationCoun) {} + void SetOptions(Benchmark* b) { + if (iterationCoun_.has_value()) { + b = b->Iterations(*iterationCoun_); } - return b; + b->Unit(benchmark::kMicrosecond); } private: - size_t iterationCoun_; + std::optional iterationCoun_; }; #ifdef ENABLE_TIME_TRACKER #define TIMETRACKER(fileName) TimeTracker timeTracker(fileName); diff --git a/cpp_src/gtests/bench/ft_bench.cc b/cpp_src/gtests/bench/ft_bench.cc index 05b81d1de..02a9ff7f7 100644 --- a/cpp_src/gtests/bench/ft_bench.cc +++ b/cpp_src/gtests/bench/ft_bench.cc @@ -39,22 +39,30 @@ int main(int argc, char** argv) { if (!err.ok()) return err.code(); ::benchmark::Initialize(&argc, argv); - size_t iterationCount = -1; // count is determined by the execution time + std::optional slowIterationCount; + std::optional fastIterationCount; if (argc > 1) { try { args::ArgumentParser parser("ft_bench additional args"); - args::ValueFlag iterCountF(parser, "ITERCOUNT", "iteration count", {"iteration_count"}, args::Options::Single); + args::ValueFlag siterCountF(parser, "SITERCOUNT", "iteration count for the slow cases", {"slow_iteration_count"}, + args::Options::Single); + args::ValueFlag fiterCountF(parser, "FITERCOUNT", "iteration count for the fast cases", {"fast_iteration_count"}, + args::Options::Single); parser.ParseCLI(argc, argv); - if (iterCountF) { - iterationCount = args::get(iterCountF); - argc--; // only one additional argument, otherwise need to rearrange the argv rows + if (siterCountF) { + slowIterationCount = args::get(siterCountF); + argc--; // sub argument, otherwise need to rearrange the argv rows + } + if (fiterCountF) { + fastIterationCount = args::get(fiterCountF); + argc--; // sub additional argument, otherwise need to rearrange the argv rows } } catch (const args::ParseError& e) { std::cout << "argument parse error '" << e.what() << "'" << std::endl; return 1; } } - ft.RegisterAllCases(iterationCount); + ft.RegisterAllCases(fastIterationCount, slowIterationCount); #ifdef _GLIBCXX_DEBUG ::benchmark::RunSpecifiedBenchmarks(); diff --git a/cpp_src/gtests/tests/API/api.cc b/cpp_src/gtests/tests/API/api.cc index bdb97049f..cf14719c5 100644 --- a/cpp_src/gtests/tests/API/api.cc +++ b/cpp_src/gtests/tests/API/api.cc @@ -1,9 +1,19 @@ #include #include "gtest/gtest.h" +#include "tools/assertrx.h" +#include "tools/fsops.h" -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { srand(time(NULL)); ::testing::InitGoogleTest(&argc, argv); + +#ifndef _WIN32 + const char *tmpDir = getenv("REINDEXER_TEST_DB_ROOT"); + if (tmpDir && *tmpDir) { + reindexer::fs::SetTempDir(std::string(tmpDir)); + } +#endif // _WIN32 + return RUN_ALL_TESTS(); } diff --git a/cpp_src/gtests/tests/API/base_tests.cc b/cpp_src/gtests/tests/API/base_tests.cc index 1f1751cd0..6586a0c67 100644 --- a/cpp_src/gtests/tests/API/base_tests.cc +++ b/cpp_src/gtests/tests/API/base_tests.cc @@ -21,8 +21,6 @@ #include "server/loggerwrapper.h" #include "tools/serializer.h" -static const std::string kBaseTestsStoragePath = "/tmp/reindex/base_tests"; - TEST_F(ReindexerApi, AddNamespace) { auto err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); ASSERT_EQ(true, err.ok()) << err.what(); @@ -507,6 +505,7 @@ TEST_F(ReindexerApi, CloseNamespace) { } TEST_F(ReindexerApi, DropStorage) { + const std::string kBaseTestsStoragePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/api_drop_storage/"); auto rx = std::make_unique(); auto err = rx->Connect("builtin://" + kBaseTestsStoragePath); ASSERT_TRUE(err.ok()) << err.what(); @@ -1613,7 +1612,7 @@ TEST_F(ReindexerApi, LoggerWriteInterruptTest) { spdlog::drop_all(); std::remove(logFile.c_str()); } - const std::string logFile = "/tmp/logtest.out"; + const std::string logFile = "/tmp/reindex_logtest.out"; reindexer_server::LoggerWrapper logger; std::shared_ptr sinkPtr; } instance; diff --git a/cpp_src/gtests/tests/fixtures/clientsstats_api.h b/cpp_src/gtests/tests/fixtures/clientsstats_api.h index 7df859ab9..87e70d4dd 100644 --- a/cpp_src/gtests/tests/fixtures/clientsstats_api.h +++ b/cpp_src/gtests/tests/fixtures/clientsstats_api.h @@ -25,7 +25,7 @@ class ClientsStatsApi : public ::testing::Test { void SetProfilingFlag(bool val, const std::string& column, reindexer::client::CoroReindexer& c); - const std::string kdbPath = "/tmp/testdb/"; + const std::string kdbPath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "clientstats_test"); const std::string kdbName = "test"; const std::string kipaddress = "127.0.0.1"; const uint16_t kPortI = 7777; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/index.cc b/cpp_src/gtests/tests/fixtures/fuzzing/index.cc index cb9f50d4e..17f7f9c59 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/index.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/index.cc @@ -13,7 +13,7 @@ reindexer::IndexDef Index::IndexDef(RandomGenerator& rnd, const NsScheme& scheme IndexOpts opts; const bool pk = rnd.PkIndex(isPk_); opts.PK(pk); - opts.Array(rnd.RndArrayField(isArray_) == IsArray::Yes); + opts.Array(rnd.RndArrayField(isArray_) == IsArrayT::Yes); opts.Sparse(rnd.RndSparseIndex(isSparse_)); opts.Dense(rnd.DenseIndex()); opts.RTreeType(static_cast(rnd.RndInt(IndexOpts::Linear, IndexOpts::RStar))); @@ -58,7 +58,7 @@ void Index::Dump(std::ostream& os, const NsScheme& scheme, size_t offset) const for (size_t i = 0; i <= offset; ++i) os << " "; os << "array: " << std::boolalpha << IsArray() << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "sparse: " << std::boolalpha << (IsSparse() == IsSparse::Yes) << '\n'; + os << "sparse: " << std::boolalpha << (IsSparse() == IsSparseT::Yes) << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; std::visit(reindexer::overloaded{[&](const Child& child) { os << "composite: false\n"; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/index.h b/cpp_src/gtests/tests/fixtures/fuzzing/index.h index 80407211c..4fe90a47b 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/index.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/index.h @@ -22,9 +22,9 @@ class Index { }; using Children = std::vector; - Index(std::string name, IndexType type, IsArray isArray, IsSparse isSparse, Children content) noexcept + Index(std::string name, IndexType type, IsArrayT isArray, IsSparseT isSparse, Children content) noexcept : name_{std::move(name)}, type_{type}, content_{std::move(content)}, isArray_{isArray}, isSparse_{isSparse} {} - Index(std::string name, IndexType type, IsArray isArray, IsSparse isSparse, Child content) noexcept + Index(std::string name, IndexType type, IsArrayT isArray, IsSparseT isSparse, Child content) noexcept : name_{std::move(name)}, type_{type}, content_{std::move(content)}, isArray_{isArray}, isSparse_{isSparse} {} const std::string& Name() const& noexcept { return name_; } @@ -34,7 +34,7 @@ class Index { const auto& Content() const&& = delete; bool IsPk() const noexcept { return isPk_; } void SetPk() noexcept { isPk_ = true; } - bool IsArray() const noexcept { return isArray_ == IsArray::Yes; } + bool IsArray() const noexcept { return isArray_ == IsArrayT::Yes; } auto IsSparse() const noexcept { return isSparse_; } reindexer::IndexDef IndexDef(RandomGenerator&, const NsScheme&, const std::vector&) const; @@ -46,8 +46,8 @@ class Index { IndexType type_; std::variant content_; bool isPk_{false}; - enum IsArray isArray_ { IsArray::No }; - enum IsSparse isSparse_ { IsSparse::No }; + IsArrayT isArray_{IsArrayT::No}; + IsSparseT isSparse_{IsSparseT::No}; }; } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc b/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc index 5a520d4a0..831653a84 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc @@ -99,7 +99,7 @@ Ns::Ns(std::string name, RandomGenerator::ErrFactorType errorFactor) usedIndexNames.insert(name); } - indexes_.emplace_back(std::move(name), indexType, rndGen_.RndArrayField(array ? IsArray::Yes : IsArray::No), IsSparse::No, + indexes_.emplace_back(std::move(name), indexType, rndGen_.RndArrayField(array ? IsArrayT::Yes : IsArrayT::No), IsSparseT::No, std::move(children)); } else { FieldPath fldPath; @@ -113,8 +113,8 @@ Ns::Ns(std::string name, RandomGenerator::ErrFactorType errorFactor) if (!rndGen_.RndErr()) continue; const auto fldType = rndGen_.RndFieldType(); indexes_.emplace_back(rndGen_.IndexName(usedIndexNames), rndGen_.RndIndexType({fldType}), - rndGen_.RndBool(0.5) ? IsArray::Yes : IsArray::No, - rndGen_.RndBool(0.5) ? IsSparse::Yes : IsSparse::No, Index::Child{fldType, std::move(fldPath)}); + rndGen_.RndBool(0.5) ? IsArrayT::Yes : IsArrayT::No, + rndGen_.RndBool(0.5) ? IsSparseT::Yes : IsSparseT::No, Index::Child{fldType, std::move(fldPath)}); } else { const auto fldType = scheme_.GetFieldType(fldPath); const auto isArray = scheme_.IsArray(fldPath); @@ -132,7 +132,7 @@ Ns::Ns(std::string name, RandomGenerator::ErrFactorType errorFactor) rndGen_.RndSparseIndex(fldType), Index::Child{fldType, std::move(fldPath)}); } if (const auto& idx = indexes_.back(); - !idx.IsArray() && idx.IsSparse() == IsSparse::No && + !idx.IsArray() && idx.IsSparse() == IsSparseT::No && std::get(idx.Content()).type != FieldType::Point) { // TODO remove point check after #1352 scalarIndexes.push_back(indexes_.size() - 1); } @@ -145,7 +145,7 @@ Ns::Ns(std::string name, RandomGenerator::ErrFactorType errorFactor) std::vector ii; for (size_t i = 0, s = indexes_.size(); i < s; ++i) { const auto& idx = indexes_[i]; - if (!idx.IsArray() && idx.IsSparse() == IsSparse::No && availablePkIndexType(idx.Type()) && + if (!idx.IsArray() && idx.IsSparse() == IsSparseT::No && availablePkIndexType(idx.Type()) && (std::holds_alternative(idx.Content()) || availablePkFieldType(std::get(idx.Content()).type))) { ii.push_back(i); } @@ -163,7 +163,7 @@ Ns::Ns(std::string name, RandomGenerator::ErrFactorType errorFactor) usedIndexNames.insert(name); } } - indexes_.emplace_back(std::move(name), rndGen_.RndPkIndexType({fldType}), IsArray::No, IsSparse::No, + indexes_.emplace_back(std::move(name), rndGen_.RndPkIndexType({fldType}), IsArrayT::No, IsSparseT::No, Index::Child{fldType, std::move(path)}); indexes_.back().SetPk(); } else { diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc index 8c027dbc6..0a1c92829 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc @@ -77,13 +77,13 @@ size_t NsScheme::FieldsCount(const FieldPath& path) const noexcept { ref[path.back()].content); } -IsArray NsScheme::IsArray(const FieldPath& path) const noexcept { +IsArrayT NsScheme::IsArray(const FieldPath& path) const noexcept { if (path.empty()) return ns_.array; const Node::Children* ptr = &std::get(ns_.content); for (size_t i = 0, s = path.size() - 1; i < s; ++i) { assertrx(ptr->size() > path[i]); const auto& idx = (*ptr)[path[i]]; - if (idx.array == IsArray::Yes) return IsArray::Yes; + if (idx.array == IsArrayT::Yes) return IsArrayT::Yes; std::visit( reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, [](const Node::Child&) noexcept { assertrx(0); }}, idx.content); @@ -127,16 +127,16 @@ std::string NsScheme::GetJsonPath(const FieldPath& path) const noexcept { return res; } -void NsScheme::AddIndex(const FieldPath& path, size_t index, IsSparse isSparse) { +void NsScheme::AddIndex(const FieldPath& path, size_t index, IsSparseT isSparse) { assertrx(!path.empty()); - if (isSparse == IsSparse::No) { - ns_.sparse = IsSparse::No; + if (isSparse == IsSparseT::No) { + ns_.sparse = IsSparseT::No; } Node::Children* ptr = &std::get(ns_.content); for (size_t i = 0, s = path.size() - 1; i < s; ++i) { assertrx(ptr->size() > path[i]); - if (isSparse == IsSparse::No) { - (*ptr)[path[i]].sparse = IsSparse::No; + if (isSparse == IsSparseT::No) { + (*ptr)[path[i]].sparse = IsSparseT::No; } std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assertrx(0); }}, (*ptr)[path[i]].content); @@ -148,14 +148,14 @@ void NsScheme::AddIndex(const FieldPath& path, size_t index, IsSparse isSparse) FieldPath NsScheme::AddRndPkField(RandomGenerator& rnd) { auto& children = std::get(ns_.content); children.emplace_back(Node{rnd.FieldName(generatedNames_), Node::Child{rnd.RndPkIndexFieldType()}}); - children.back().array = IsArray::No; - children.back().sparse = IsSparse::No; + children.back().array = IsArrayT::No; + children.back().sparse = IsSparseT::No; return {children.size() - 1}; } -void NsScheme::addIndex(Node& node, size_t index, IsSparse isSparse) { - if (isSparse == IsSparse::No) { - node.sparse = IsSparse::No; +void NsScheme::addIndex(Node& node, size_t index, IsSparseT isSparse) { + if (isSparse == IsSparseT::No) { + node.sparse = IsSparseT::No; } std::visit(reindexer::overloaded{[index](Node::Child& c) noexcept { c.indexes.push_back(index); }, [](Node::Children&) noexcept { assertrx(0); }}, @@ -175,14 +175,14 @@ void NsScheme::fillChildren(Node::Children& children, RandomGenerator& rnd, unsi children.back().array = rnd.RndArrayField(); } if (!canBeSparse && !rnd.RndErr()) { - children.back().sparse = IsSparse::No; + children.back().sparse = IsSparseT::No; } } else { children.emplace_back(Node{std::move(fName), Node::Child{type}}); if (type == FieldType::Point) { canBeSparse = false; canBeArray = false; - children.back().sparse = IsSparse::No; + children.back().sparse = IsSparseT::No; } if (canBeArray || rnd.RndErr()) { children.back().array = rnd.RndArrayField(); @@ -256,7 +256,7 @@ void NsScheme::toJson(reindexer::JsonBuilder& builder, const Node::Children& chi const std::vector& indexes) { for (const Node& n : children) { if (!rnd.NeedThisNode(n.sparse)) continue; - if (rnd.RndArrayField(n.array) == IsArray::Yes) { + if (rnd.RndArrayField(n.array) == IsArrayT::Yes) { auto arr = builder.Array(n.name); const size_t arrSize = rnd.ArraySize(); for (size_t i = 0; i < arrSize; ++i) { @@ -294,9 +294,9 @@ void NsScheme::Node::Dump(std::ostream& os, size_t offset) const { for (size_t i = 0; i <= offset; ++i) os << " "; os << "name: " << name << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "sparse: " << std::boolalpha << (sparse == IsSparse::Yes) << '\n'; + os << "sparse: " << std::boolalpha << (sparse == IsSparseT::Yes) << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "array: " << std::boolalpha << (array == IsArray::Yes) << '\n'; + os << "array: " << std::boolalpha << (array == IsArrayT::Yes) << '\n'; std::visit(reindexer::overloaded{[&](const Child& child) { for (size_t i = 0; i <= offset; ++i) os << " "; os << "type: " << child.type << '\n'; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h index a63a76e38..8123d726f 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h @@ -29,8 +29,8 @@ class NsScheme { std::string name; std::variant content; - IsSparse sparse{IsSparse::Yes}; - IsArray array{IsArray::No}; + IsSparseT sparse{IsSparseT::Yes}; + IsArrayT array{IsArrayT::No}; void Dump(std::ostream&, size_t offset) const; }; @@ -43,17 +43,17 @@ class NsScheme { bool IsStruct(const FieldPath&) const noexcept; bool IsPoint(const FieldPath&) const noexcept; bool IsTtl(const FieldPath&, const std::vector&) const noexcept; - enum IsArray IsArray(const FieldPath&) const noexcept; + IsArrayT IsArray(const FieldPath&) const noexcept; FieldType GetFieldType(const FieldPath&) const noexcept; void SetFieldType(const FieldPath&, FieldType) noexcept; std::string GetJsonPath(const FieldPath&) const noexcept; - void AddIndex(const FieldPath&, size_t index, IsSparse); + void AddIndex(const FieldPath&, size_t index, IsSparseT); void NewItem(reindexer::WrSerializer&, RandomGenerator&, const std::vector&); void Dump(std::ostream& os, size_t offset) const { ns_.Dump(os, offset); } FieldPath AddRndPkField(RandomGenerator&); private: - static void addIndex(Node&, size_t index, IsSparse); + static void addIndex(Node&, size_t index, IsSparseT); void fillChildren(Node::Children&, RandomGenerator&, unsigned level, bool& canBeArray, bool& canBeSparse); const Node::Children& findLastContainer(const FieldPath&) const noexcept; Node::Children& findLastContainer(const FieldPath&) noexcept; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc index e207a6667..82d85a68e 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc @@ -228,7 +228,7 @@ FieldPath RandomGenerator::RndScalarField(const NsScheme& nsScheme) { const int end = idx + size; while (idx < end) { res.back() = idx % size; - if (nsScheme.IsArray(res) == IsArray::No && !nsScheme.IsPoint(res)) break; + if (nsScheme.IsArray(res) == IsArrayT::No && !nsScheme.IsPoint(res)) break; ++idx; } if (idx == end) return {}; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h index 4960f28c3..bebb70490 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h @@ -60,20 +60,20 @@ class RandomGenerator { IndexType RndIndexType(const std::vector&); IndexType RndPkIndexType(const std::vector&); IndexType RndIndexType(IndexType); - IsArray RndArrayField() { return RndBool(0.2) ? IsArray::Yes : IsArray::No; } - IsArray RndArrayField(IsArray array) { + IsArrayT RndArrayField() { return RndBool(0.2) ? IsArrayT::Yes : IsArrayT::No; } + IsArrayT RndArrayField(IsArrayT array) { if (RndErr()) { - return array == IsArray::Yes ? IsArray::No : IsArray::Yes; + return array == IsArrayT::Yes ? IsArrayT::No : IsArrayT::Yes; } return array; } size_t ArraySize(); bool PkIndex(bool pk) { return RndErr() ? RndBool(0.5) : pk; } - IsSparse RndSparseIndex(FieldType fldType) { + IsSparseT RndSparseIndex(FieldType fldType) { const bool couldBeSparse = fldType != FieldType::Struct && fldType != FieldType::Uuid; // TODO remove uuid #1470 - return (couldBeSparse ? RndBool(0.2) : RndErr()) ? IsSparse::Yes : IsSparse::No; + return (couldBeSparse ? RndBool(0.2) : RndErr()) ? IsSparseT::Yes : IsSparseT::No; } - bool RndSparseIndex(IsSparse isSparse) { return (isSparse == IsSparse::Yes) != RndErr(); } + bool RndSparseIndex(IsSparseT isSparse) { return (isSparse == IsSparseT::Yes) != RndErr(); } bool DenseIndex() { return RndBool(0.2); } int64_t ExpiredIndex() { return RndInt(0, 100'000); } // TODO size_t IndexesCount(); @@ -137,7 +137,7 @@ class RandomGenerator { return err; } char RndChar() { return rndChar_(gen_); } - bool NeedThisNode(IsSparse sparse) { return sparse == IsSparse::Yes ? RndBool(0.5) : !RndErr(); } + bool NeedThisNode(IsSparseT sparse) { return sparse == IsSparseT::Yes ? RndBool(0.5) : !RndErr(); } int RndIntValue() { enum Size : uint8_t { Short, Long, END = Long }; switch (RndWhich()) { diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/types.h b/cpp_src/gtests/tests/fixtures/fuzzing/types.h index db97087c6..98ec500a8 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/types.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/types.h @@ -23,7 +23,7 @@ enum class IndexType { Store, Hash, Tree, Ttl, FastFT, FuzzyFT, RTree, END = RTr std::string_view ToText(IndexType); std::ostream& operator<<(std::ostream&, IndexType); -enum class IsArray : bool { Yes = true, No = false }; -enum class IsSparse : bool { Yes = true, No = false }; +enum class IsArrayT : bool { Yes = true, No = false }; +enum class IsSparseT : bool { Yes = true, No = false }; } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/grpcclient_api.h b/cpp_src/gtests/tests/fixtures/grpcclient_api.h index 98b05c866..7a772dcd9 100644 --- a/cpp_src/gtests/tests/fixtures/grpcclient_api.h +++ b/cpp_src/gtests/tests/fixtures/grpcclient_api.h @@ -17,6 +17,7 @@ class GrpcClientApi : public ReindexerApi { public: void SetUp() { + reindexer::fs::RmDirAll(kStoragePath); YAML::Node y; y["storage"]["path"] = kStoragePath; y["logger"]["loglevel"] = "none"; @@ -27,7 +28,6 @@ class GrpcClientApi : public ReindexerApi { y["net"]["rpcaddr"] = "0.0.0.0:4443"; y["net"]["grpc"] = true; - reindexer::fs::RmDirAll(kStoragePath); auto err = srv_.InitFromYAML(YAML::Dump(y)); EXPECT_TRUE(err.ok()) << err.what(); @@ -42,7 +42,7 @@ class GrpcClientApi : public ReindexerApi { std::this_thread::sleep_for(std::chrono::milliseconds(20)); } - for (int i = 0; i < 5; i++) { + for (int i = 0; i < 10; i++) { auto channel = grpc::CreateChannel("127.0.0.1:16534", grpc::InsecureChannelCredentials()); if (channel->WaitForConnected(std::chrono::system_clock::now() + std::chrono::seconds(1))) { rx_ = reindexer::grpc::Reindexer::NewStub(channel); @@ -351,7 +351,7 @@ class GrpcClientApi : public ReindexerApi { std::unique_ptr rx_; uint64_t lastLsn_ = 0, lastRowId_ = INT_MAX; - const std::string kStoragePath = "/tmp/reindex_grpc_test/"; + const std::string kStoragePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex_grpc_test"); }; #endif diff --git a/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h b/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h index 8834ba94e..5173b168d 100644 --- a/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h +++ b/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h @@ -4,7 +4,7 @@ class JoinOnConditionsApi : public JoinSelectsApi { public: - void SetUp() override { JoinSelectsApi::Init("/tmp/join_on_conditions_test/"); } + void SetUp() override { JoinSelectsApi::Init(reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "join_on_conditions_test")); } void CreateCondSetTable(const std::string& leftNs, const std::string& rightNs, const std::vector& leftNsData, const std::vector>& rightNsData) { diff --git a/cpp_src/gtests/tests/fixtures/join_selects_api.h b/cpp_src/gtests/tests/fixtures/join_selects_api.h index cdf0f9189..b9e693312 100644 --- a/cpp_src/gtests/tests/fixtures/join_selects_api.h +++ b/cpp_src/gtests/tests/fixtures/join_selects_api.h @@ -19,7 +19,7 @@ class JoinSelectsApi : public ReindexerApi { using QueryResultRow = std::map; using QueryResultRows = std::map; - void Init(const std::string& dbName = "/tmp/join_test/") { + void Init(const std::string& dbName = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "join_test")) { Error err; reindexer::fs::RmDirAll(dbName); diff --git a/cpp_src/gtests/tests/fixtures/msgpack_cproto_api.h b/cpp_src/gtests/tests/fixtures/msgpack_cproto_api.h index ce7f7b985..6661f5867 100644 --- a/cpp_src/gtests/tests/fixtures/msgpack_cproto_api.h +++ b/cpp_src/gtests/tests/fixtures/msgpack_cproto_api.h @@ -20,7 +20,7 @@ class MsgPackCprotoApi : public ReindexerApi { void SetUp() { reindexer::fs::RmDirAll(kDbPath); YAML::Node y; - y["storage"]["path"] = "/tmp/reindex/" + kDbPath; + y["storage"]["path"] = kDbPath; y["logger"]["loglevel"] = "none"; y["logger"]["rpclog"] = "none"; y["logger"]["serverlog"] = "none"; @@ -40,7 +40,7 @@ class MsgPackCprotoApi : public ReindexerApi { } client_.reset(new reindexer::client::RPCClientMock()); - err = client_->Connect("cproto://127.0.0.1:25677/" + kDbPath, reindexer::client::ConnectOpts().CreateDBIfMissing()); + err = client_->Connect("cproto://127.0.0.1:25677/" + kDbName, reindexer::client::ConnectOpts().CreateDBIfMissing()); ASSERT_TRUE(err.ok()) << err.what(); err = client_->OpenNamespace(default_namespace, ctx_, StorageOpts().CreateIfMissing()); @@ -81,11 +81,12 @@ class MsgPackCprotoApi : public ReindexerApi { void TearDown() { if (server_.IsRunning()) { server_.Stop(); - if (serverThread_->joinable()) { - serverThread_->join(); - } } - client_->Stop(); + client_.reset(); + if (serverThread_->joinable()) { + serverThread_->join(); + } + serverThread_.reset(); } void checkItem(reindexer::client::QueryResults::Iterator& it) { @@ -105,7 +106,8 @@ class MsgPackCprotoApi : public ReindexerApi { } protected: - const std::string kDbPath = "cproto_msgpack_test"; + const std::string kDbName = "cproto_msgpack_test"; + const std::string kDbPath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/" + kDbName); const std::string kFieldId = "id"; const std::string kFieldA1 = "a1"; const std::string kFieldA2 = "a2"; diff --git a/cpp_src/gtests/tests/fixtures/queries_api.h b/cpp_src/gtests/tests/fixtures/queries_api.h index ed95e3a38..0b3883729 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.h +++ b/cpp_src/gtests/tests/fixtures/queries_api.h @@ -1880,12 +1880,13 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { using namespace std::string_literals; using reindexer::randPoint; using reindexer::randBinDouble; + using reindexer::double_to_str; // ---------- reindexer::Point point{randPoint(10)}; double distance = randBinDouble(0, 1); std::string dslQuery = fmt::sprintf( - R"({"namespace":"%s","limit":-1,"offset":0,"req_total":"disabled","explain":false,"type":"select","select_with_rank":false,"select_filter":[],"select_functions":[],"sort":[],"filters":[{"op":"and","cond":"dwithin","field":"%s","value":[[%f, %f], %f]}],"merge_queries":[],"aggregations":[]})", - geomNs, kFieldNamePointLinearRTree, point.X(), point.Y(), distance); + R"({"namespace":"%s","limit":-1,"offset":0,"req_total":"disabled","explain":false,"type":"select","select_with_rank":false,"select_filter":[],"select_functions":[],"sort":[],"filters":[{"op":"and","cond":"dwithin","field":"%s","value":[[%s, %s], %s]}],"merge_queries":[],"aggregations":[]})", + geomNs, kFieldNamePointLinearRTree, double_to_str(point.X()), double_to_str(point.Y()), double_to_str(distance)); const Query checkQuery1{Query(geomNs).DWithin(kFieldNamePointLinearRTree, point, distance)}; checkDslQuery(dslQuery, checkQuery1); @@ -1893,8 +1894,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { point = randPoint(10); distance = randBinDouble(0, 1); dslQuery = fmt::sprintf( - R"({"namespace":"%s","limit":-1,"offset":0,"req_total":"disabled","explain":false,"type":"select","select_with_rank":false,"select_filter":[],"select_functions":[],"sort":[],"filters":[{"op":"and","cond":"dwithin","field":"%s","value":[%f,[%f,%f]]}],"merge_queries":[],"aggregations":[]})", - geomNs, kFieldNamePointLinearRTree, distance, point.X(), point.Y()); + R"({"namespace":"%s","limit":-1,"offset":0,"req_total":"disabled","explain":false,"type":"select","select_with_rank":false,"select_filter":[],"select_functions":[],"sort":[],"filters":[{"op":"and","cond":"dwithin","field":"%s","value":[%s,[%s,%s]]}],"merge_queries":[],"aggregations":[]})", + geomNs, kFieldNamePointLinearRTree, double_to_str(distance), double_to_str(point.X()), double_to_str(point.Y())); const Query checkQuery2{Query(geomNs).DWithin(kFieldNamePointLinearRTree, point, distance)}; checkDslQuery(dslQuery, checkQuery2); diff --git a/cpp_src/gtests/tests/fixtures/queries_verifier.h b/cpp_src/gtests/tests/fixtures/queries_verifier.h index ed717fdca..3d0b9b684 100644 --- a/cpp_src/gtests/tests/fixtures/queries_verifier.h +++ b/cpp_src/gtests/tests/fixtures/queries_verifier.h @@ -305,7 +305,7 @@ class QueriesVerifier : public virtual ::testing::Test { bool result = true; // check only on root level for (auto it = qr.entries.cbegin(); it != qr.entries.cend(); ++it) { - if (!it->HoldsOrReferTo()) continue; + if (!it->Is()) continue; const reindexer::QueryEntry& qentry = it->Value(); if (!qentry.Distinct()) continue; diff --git a/cpp_src/gtests/tests/fixtures/replication_api.cc b/cpp_src/gtests/tests/fixtures/replication_api.cc index 66ca8a4aa..f37846a54 100644 --- a/cpp_src/gtests/tests/fixtures/replication_api.cc +++ b/cpp_src/gtests/tests/fixtures/replication_api.cc @@ -6,8 +6,6 @@ #include "tools/fsops.h" #include "vendor/gason/gason.h" -const std::string ReplicationApi::kStoragePath = "/tmp/reindex_repl_test/"; -// const std::string ReplicationApi::kReplicationConfigFilename = "replication.conf"; const std::string ReplicationApi::kConfigNs = "#config"; bool ReplicationApi::StopServer(size_t id) { diff --git a/cpp_src/gtests/tests/fixtures/replication_api.h b/cpp_src/gtests/tests/fixtures/replication_api.h index 455715321..19e2cf9ce 100644 --- a/cpp_src/gtests/tests/fixtures/replication_api.h +++ b/cpp_src/gtests/tests/fixtures/replication_api.h @@ -23,8 +23,6 @@ const auto kMaxForceSyncCmdTime = std::chrono::seconds(10); class ReplicationApi : public ::testing::Test { public: - static const std::string kStoragePath; - static const std::string kReplicationConfigFilename; static const std::string kConfigNs; void SetUp(); @@ -55,6 +53,7 @@ class ReplicationApi : public ::testing::Test { shared_timed_mutex restartMutex_; private: + const std::string kStoragePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex_repl_test/"); std::vector svc_; mutable std::mutex m_; }; diff --git a/cpp_src/gtests/tests/fixtures/replication_load_api.h b/cpp_src/gtests/tests/fixtures/replication_load_api.h index 749f11ebf..ddbbf1fa2 100644 --- a/cpp_src/gtests/tests/fixtures/replication_load_api.h +++ b/cpp_src/gtests/tests/fixtures/replication_load_api.h @@ -130,7 +130,7 @@ class ReplicationLoadApi : public ReplicationApi { assertrx(num); auto srv = GetSrv(num); auto curConfig = srv->GetServerConfig(ServerControl::ConfigType::File); - EXPECT_TRUE(config == curConfig); + EXPECT_TRUE(config == curConfig) << "config:\n" << config.GetJSON() << "\ncurConfig:\n" << curConfig.GetJSON(); } void CheckSlaveConfigNamespace(size_t num, const ReplicationConfigTest &config, std::chrono::seconds awaitTime) { assertrx(num); @@ -142,7 +142,8 @@ class ReplicationLoadApi : public ReplicationApi { } std::this_thread::sleep_for(std::chrono::seconds(1)); } - EXPECT_TRUE(config == srv->GetServerConfig(ServerControl::ConfigType::Namespace)); + auto curConfig = srv->GetServerConfig(ServerControl::ConfigType::Namespace); + EXPECT_TRUE(config == curConfig) << "config:\n" << config.GetJSON() << "\ncurConfig:\n" << curConfig.GetJSON(); } std::atomic_bool stop; diff --git a/cpp_src/gtests/tests/fixtures/rpcclient_api.h b/cpp_src/gtests/tests/fixtures/rpcclient_api.h index 5edb3ffd5..470331401 100644 --- a/cpp_src/gtests/tests/fixtures/rpcclient_api.h +++ b/cpp_src/gtests/tests/fixtures/rpcclient_api.h @@ -7,6 +7,7 @@ #include "client/reindexer.h" #include "replicator/updatesobserver.h" #include "server/server.h" +#include "tools/fsops.h" #include "client/cororeindexer.h" @@ -107,7 +108,7 @@ class RPCClientTestApi : public ::testing::Test { void FillData(reindexer::client::Reindexer& rx, std::string_view nsName, int from, int count); void FillData(reindexer::client::CoroReindexer& rx, std::string_view nsName, int from, int count); - const std::string_view kDbPrefix{"/tmp/reindex/rpc_client_test"}; + const std::string kDbPrefix{reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/rpc_client_test")}; private: struct ServerData { diff --git a/cpp_src/gtests/tests/fixtures/servercontrol.cc b/cpp_src/gtests/tests/fixtures/servercontrol.cc index bcbfc80f2..718de3ad9 100644 --- a/cpp_src/gtests/tests/fixtures/servercontrol.cc +++ b/cpp_src/gtests/tests/fixtures/servercontrol.cc @@ -12,6 +12,27 @@ using namespace reindexer; +std::string ReplicationConfigTest::GetJSON() const { + WrSerializer ser; + { + JsonBuilder builder(ser); + + builder.Put("role", role_); + builder.Put("server_id", serverId_); + builder.Put("dsn", dsn_); + builder.Put("app_name", appName_); + builder.Put("force_sync_on_logic_error", forceSyncOnLogicError_); + builder.Put("force_sync_on_wrong_datahash", forceSyncOnWrongDataHash_); + { + auto jsn = builder.Array("namespaces"); + for (auto& ns : namespaces_) { + jsn.Put(nullptr, ns); + } + } + } + return std::string(ser.Slice()); +} + ServerControl::Interface::~Interface() { Stop(); if (tr) { diff --git a/cpp_src/gtests/tests/fixtures/servercontrol.h b/cpp_src/gtests/tests/fixtures/servercontrol.h index 86a4154ab..c034588a3 100644 --- a/cpp_src/gtests/tests/fixtures/servercontrol.h +++ b/cpp_src/gtests/tests/fixtures/servercontrol.h @@ -9,6 +9,7 @@ #include "estl/shared_mutex.h" #include "reindexertestapi.h" #include "server/server.h" +#include "tools/fsops.h" #include "tools/stringstools.h" #ifdef REINDEXER_WITH_SC_AS_PROCESS @@ -44,6 +45,8 @@ struct ReplicationConfigTest { namespaces_ == config.namespaces_ && serverId_ == config.serverId_; } + std::string GetJSON() const; + std::string role_; bool forceSyncOnLogicError_; bool forceSyncOnWrongDataHash_; @@ -67,7 +70,7 @@ class ServerControl { public: const std::string kReplicationConfigFilename = "replication.conf"; const std::string kConfigNs = "#config"; - const std::string kStoragePath = "/tmp/reindex_repl_test/"; + const std::string kStoragePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex_repl_test"); const unsigned short kDefaultHttpPort = 5555; const size_t kMaxServerStartTimeSec = 20; diff --git a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h index c32ec7caf..bc4bfe2a3 100644 --- a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h +++ b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h @@ -106,7 +106,7 @@ class DISABLED_StorageLazyLoadApi : public ReindexerApi { const char* kFieldId = "id"; const char* kFieldRandomName = "random_name"; const char* kConfigNamespace = "#config"; - const std::string kStoragePath = "/tmp/reindex/lazy_load_test"; + const std::string kStoragePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/lazy_load_test"); const char* jsonConfigTemplate = R"json({ "type":"namespaces", "namespaces":[ diff --git a/cpp_src/gtests/tests/unit/namespace_test.cc b/cpp_src/gtests/tests/unit/namespace_test.cc index d0b85f90d..e6a4fc5ad 100644 --- a/cpp_src/gtests/tests/unit/namespace_test.cc +++ b/cpp_src/gtests/tests/unit/namespace_test.cc @@ -494,6 +494,44 @@ TEST_F(NsApi, TestUpdateTwoFields) { } } +TEST_F(NsApi, TestUpdateNewFieldCheckTmVersion) { + DefineDefaultNamespace(); + FillDefaultNamespace(); + + auto check = [this](const Query &query, int tmVersion) { + QueryResults qrUpdate; + auto err = rt.reindexer->Update(query, qrUpdate); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qrUpdate.Count(), 1); + ASSERT_EQ(qrUpdate.getTagsMatcher(0).version(), tmVersion); + }; + + QueryResults qr; + Error err = rt.reindexer->Select(Query(default_namespace).Where(idIdxName, CondEq, 1), qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.Count(), 1); + auto tmVersion = qr.getTagsMatcher(0).version(); + Query updateQuery = Query(default_namespace).Where(idIdxName, CondEq, 1).Set("some_new_field", "some_value"); + + // Make sure the version increases by 1 when one new tag with non-object content is added + check(updateQuery, ++tmVersion); + + // Make sure the version not change when the same update query applied + check(updateQuery, tmVersion); + + updateQuery.SetObject("new_obj_field", R"({"id":111, "name":"successfully updated!"})"); + + // Make sure that tm version updates correctly when new tags are added: + // +1 by tag very_nested, + // +1 by all new tags processed in ItemModifier::modifyCJSON for SetObject-method + // +1 by merge of two corresponded tagsmatchers + check(updateQuery, tmVersion += 3); + + // Make sure that if no new tags were added to the tagsmatcher during the update, + // then the version of the tagsmatcher will not change + check(updateQuery, tmVersion); +} + static void updateArrayField(const std::shared_ptr &reindexer, const std::string &ns, const std::string &updateFieldPath, const VariantArray &values) { QueryResults qrUpdate; @@ -1849,7 +1887,7 @@ TEST_F(NsApi, TestUpdatePkFieldNoConditions) { ASSERT_TRUE(err.ok()) << err.what(); QueryResults qr; - err = rt.reindexer->Select("update test_namespace set id = id + "+std::to_string(qrCount.totalCount+100), qr); + err = rt.reindexer->Select("update test_namespace set id = id + " + std::to_string(qrCount.totalCount + 100), qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_GT(qr.Count(), 0); @@ -1857,7 +1895,7 @@ TEST_F(NsApi, TestUpdatePkFieldNoConditions) { for (auto &it : qr) { Item item = it.GetItem(false); Variant intFieldVal = item[idIdxName]; - ASSERT_EQ(static_cast(intFieldVal) , i+qrCount.totalCount+100); + ASSERT_EQ(static_cast(intFieldVal), i + qrCount.totalCount + 100); i++; } } diff --git a/cpp_src/gtests/tests/unit/replication_master_master_test.cc b/cpp_src/gtests/tests/unit/replication_master_master_test.cc index b620b6109..d3d5c81ef 100644 --- a/cpp_src/gtests/tests/unit/replication_master_master_test.cc +++ b/cpp_src/gtests/tests/unit/replication_master_master_test.cc @@ -873,16 +873,16 @@ TEST_F(ReplicationSlaveSlaveApi, Node3ApplyWal) { } TEST_F(ReplicationSlaveSlaveApi, RestrictUpdates) { - // 1. create master node, - // 2. set max updates size 1024 * 5 - // 3. add 10000 rows - // 4. start inser thread - // 5. start slave node - // 6. wait sync - reindexer::fs::RmDirAll("/tmp/RestrictUpdates"); + // 1. create master node, + // 2. set max updates size 1024 * 5 + // 3. add 10000 rows + // 4. start inser thread + // 5. start slave node + // 6. wait sync + const std::string kBaseStoragePath = reindexer::fs::JoinPath(kBaseTestsetDbPath, "RestrictUpdates"); std::string upDsn = "cproto://127.0.0.1:7770/db"; ServerControl master; - master.InitServer(0, 7770, 7880, "/tmp/RestrictUpdates/master", "db", true, 1024 * 5); + master.InitServer(0, 7770, 7880, reindexer::fs::JoinPath(kBaseStoragePath, "master"), "db", true, 1024 * 5); master.Get()->MakeMaster(ReplicationConfigTest("master", "appMaster")); TestNamespace1 testns1(master, "ns1"); @@ -906,7 +906,7 @@ TEST_F(ReplicationSlaveSlaveApi, RestrictUpdates) { std::thread insertThread(ThreadAdd); ServerControl slave; - slave.InitServer(0, 7771, 7881, "/tmp/RestrictUpdates/slave", "db", true); + slave.InitServer(0, 7771, 7881, reindexer::fs::JoinPath(kBaseStoragePath, "slave"), "db", true); ReplicationConfigTest configSlave("slave", false, true, 0, upDsn, "slave"); slave.Get()->MakeSlave(0, configSlave); diff --git a/cpp_src/gtests/tests_data/MsgPack/msg.json b/cpp_src/gtests/tests_data/MsgPack/msg.json index af8f4d5ff..2dc741879 100644 --- a/cpp_src/gtests/tests_data/MsgPack/msg.json +++ b/cpp_src/gtests/tests_data/MsgPack/msg.json @@ -119,13 +119,13 @@ "1": { "user": { "id": 19, - "price": 424, - "oldPrice": 424 + "price": 424.0, + "oldPrice": 424.0 }, "guest": { "id": 20, - "price": 424, - "oldPrice": 424 + "price": 424.0, + "oldPrice": 424.0 } } }, @@ -200,13 +200,13 @@ "1": { "user": { "id": 29, - "price": 623, - "oldPrice": 623 + "price": 623.0, + "oldPrice": 623.0 }, "guest": { "id": 30, - "price": 623, - "oldPrice": 623 + "price": 623.0, + "oldPrice": 623.0 } } }, @@ -416,13 +416,13 @@ "1": { "user": { "id": 55, - "price": 371, - "oldPrice": 371 + "price": 371.0, + "oldPrice": 371.0 }, "guest": { "id": 56, - "price": 371, - "oldPrice": 371 + "price": 371.0, + "oldPrice": 371.0 } } }, @@ -470,13 +470,13 @@ "1": { "user": { "id": 23, - "price": 371, - "oldPrice": 371 + "price": 371.0, + "oldPrice": 371.0 }, "guest": { "id": 24, - "price": 371, - "oldPrice": 371 + "price": 371.0, + "oldPrice": 371.0 } } }, @@ -630,4 +630,5 @@ "title": "12" } ] -} \ No newline at end of file +} + diff --git a/cpp_src/readme.md b/cpp_src/readme.md index 05c6e0cfb..b496cca9e 100644 --- a/cpp_src/readme.md +++ b/cpp_src/readme.md @@ -46,7 +46,7 @@ yum update yum install reindexer-server ``` -Available distros: `centos-7`, `fedora-36`, `fedora-37`, `redos-7` +Available distros: `centos-7`, `fedora-38`, `fedora-39`, `redos-7` ### Ubuntu/Debian diff --git a/cpp_src/replicator/replicator.cc b/cpp_src/replicator/replicator.cc index b44862777..93e8bdaa4 100644 --- a/cpp_src/replicator/replicator.cc +++ b/cpp_src/replicator/replicator.cc @@ -705,6 +705,20 @@ Error Replicator::applyTxWALRecord(LSNPair LSNs, std::string_view nsName, Namesp slaveNs->CommitTransaction(tx, res, rdxContext); tx = Transaction{}; } break; + case WalTagsMatcher: { + std::lock_guard lck(syncMtx_); + Transaction &tx = transactions_[slaveNs.get()]; + if (tx.IsFree()) return Error(errLogic, "[repl:%s]:%d Transaction was not initiated.", nsName, config_.serverId); + + TagsMatcher tm; + Serializer ser(rec.data.data(), rec.data.size()); + const auto version = ser.GetVarint(); + const auto stateToken = ser.GetVarint(); + tm.deserialize(ser, version, stateToken); + logPrintf(LogInfo, "[repl:%s]:%d Got new tagsmatcher replicated via tx: { state_token: %08X, version: %d }", nsName, + config_.serverId, stateToken, version); + tx.SetTagsMatcher(std::move(tm)); + } break; case WalEmpty: case WalReplState: case WalItemUpdate: @@ -718,6 +732,9 @@ Error Replicator::applyTxWALRecord(LSNPair LSNs, std::string_view nsName, Namesp case WalForceSync: case WalSetSchema: case WalWALSync: + case WalResetLocalWal: + case WalRawItem: + case WalShallowItem: return Error(errLogic, "Unexpected for transaction WAL rec type %d\n", int(rec.type)); } return {}; @@ -850,10 +867,24 @@ Error Replicator::applyWALRecord(LSNPair LSNs, std::string_view nsName, Namespac slaveNs->SetSchema(rec.data, rdxContext); stat.schemasSet++; break; + case WalTagsMatcher: { + checkNoOpenedTransaction(nsName, slaveNs); + TagsMatcher tm; + Serializer ser(rec.data.data(), rec.data.size()); + const auto version = ser.GetVarint(); + const auto stateToken = ser.GetVarint(); + tm.deserialize(ser, version, stateToken); + logPrintf(LogInfo, "[repl:%s]:%d Got new tagsmatcher replicated via single record: { state_token: %08X, version: %d }", nsName, + config_.serverId, stateToken, version); + slaveNs->SetTagsMatcher(std::move(tm), rdxContext); + } break; case WalEmpty: case WalItemUpdate: case WalInitTransaction: case WalCommitTransaction: + case WalResetLocalWal: + case WalRawItem: + case WalShallowItem: return Error(errLogic, "Unexpected WAL rec type %d\n", int(rec.type)); } return err; @@ -982,6 +1013,24 @@ Error Replicator::syncMetaForced(Namespace::Ptr &slaveNs, std::string_view nsNam // Callback from WAL updates pusher void Replicator::OnWALUpdate(LSNPair LSNs, std::string_view nsName, const WALRecord &wrec) { + try { + onWALUpdateImpl(LSNs, nsName, wrec); + } catch (Error &e) { + logPrintf(LogError, "[repl:%s]:%d Exception on WAL update: %s", nsName, config_.serverId, e.what()); + assertrx_dbg(false); + resync_.send(); + } catch (std::exception &e) { + logPrintf(LogError, "[repl:%s]:%d Exception on WAL update (std::exception): %s", nsName, config_.serverId, e.what()); + assertrx_dbg(false); + resync_.send(); + } catch (...) { + logPrintf(LogError, "[repl:%s]:%d Exception on WAL update: ", nsName, config_.serverId); + assertrx_dbg(false); + resync_.send(); + } +} + +void Replicator::onWALUpdateImpl(LSNPair LSNs, std::string_view nsName, const WALRecord &wrec) { auto sId = LSNs.originLSN_.Server(); if (sId != 0) { // sId = 0 for configurations without specifying a server id if (sId == config_.serverId) { diff --git a/cpp_src/replicator/replicator.h b/cpp_src/replicator/replicator.h index 0a2c9b4bf..5a3949a9d 100644 --- a/cpp_src/replicator/replicator.h +++ b/cpp_src/replicator/replicator.h @@ -94,7 +94,8 @@ class Replicator : public IUpdatesObserver { // Push update to the queue to apply it later void pushPendingUpdate(LSNPair LSNs, std::string_view nsName, const WALRecord &wrec); - void OnWALUpdate(LSNPair LSNs, std::string_view nsName, const WALRecord &walRec) override final; + void OnWALUpdate(LSNPair LSNs, std::string_view nsName, const WALRecord &wrec) override final; + void onWALUpdateImpl(LSNPair LSNs, std::string_view nsName, const WALRecord &wrec); void OnUpdatesLost(std::string_view nsName) override final; void OnConnectionState(const Error &err) override final; diff --git a/cpp_src/replicator/walrecord.cc b/cpp_src/replicator/walrecord.cc index e97d56757..b2222f6ec 100644 --- a/cpp_src/replicator/walrecord.cc +++ b/cpp_src/replicator/walrecord.cc @@ -19,8 +19,9 @@ void WALRecord::Pack(WrSerializer &ser) const { ser.PutVarUint(inTransaction ? (type | TxBit) : type); switch (type) { case WalItemUpdate: + case WalShallowItem: ser.PutUInt32(id); - break; + return; case WalUpdateQuery: case WalIndexAdd: case WalIndexDrop: @@ -30,29 +31,32 @@ void WALRecord::Pack(WrSerializer &ser) const { case WalForceSync: case WalWALSync: case WalSetSchema: + case WalTagsMatcher: ser.PutVString(data); - break; + return; case WalPutMeta: ser.PutVString(putMeta.key); ser.PutVString(putMeta.value); - break; + return; case WalItemModify: ser.PutVString(itemModify.itemCJson); ser.PutVarUint(itemModify.modifyMode); ser.PutVarUint(itemModify.tmVersion); - break; + return; + case WalRawItem: + ser.PutUInt32(rawItem.id); + ser.PutVString(rawItem.itemCJson); + return; case WalEmpty: - ser.Reset(); - break; case WalNamespaceAdd: case WalNamespaceDrop: case WalInitTransaction: case WalCommitTransaction: - break; - default: - fprintf(stderr, "Unexpected WAL rec type %d\n", int(type)); - std::abort(); + case WalResetLocalWal: + return; } + fprintf(stderr, "Unexpected WAL rec type %d\n", int(type)); + std::abort(); } WALRecord::WALRecord(span packed) { @@ -71,8 +75,9 @@ WALRecord::WALRecord(span packed) { } switch (type) { case WalItemUpdate: + case WalShallowItem: id = ser.GetUInt32(); - break; + return; case WalUpdateQuery: case WalIndexAdd: case WalIndexDrop: @@ -82,27 +87,31 @@ WALRecord::WALRecord(span packed) { case WalForceSync: case WalWALSync: case WalSetSchema: + case WalTagsMatcher: data = ser.GetVString(); - break; + return; case WalPutMeta: putMeta.key = ser.GetVString(); putMeta.value = ser.GetVString(); - break; + return; case WalItemModify: itemModify.itemCJson = ser.GetVString(); itemModify.modifyMode = ser.GetVarUint(); itemModify.tmVersion = ser.GetVarUint(); - break; + return; + case WalRawItem: + rawItem.id = ser.GetUInt32(); + rawItem.itemCJson = ser.GetVString(); + return; case WalEmpty: case WalNamespaceAdd: case WalNamespaceDrop: case WalInitTransaction: case WalCommitTransaction: - break; - default: - logPrintf(LogWarning, "Unexpected WAL rec type %d\n", int(type)); - break; + case WalResetLocalWal: + return; } + logPrintf(LogError, "Unexpected WAL rec type %d\n", int(type)); } static std::string_view wrecType2Str(WALRecType t) { @@ -142,9 +151,16 @@ static std::string_view wrecType2Str(WALRecType t) { return "WalWALSync"sv; case WalSetSchema: return "WalSetSchema"sv; - default: - return ""sv; + case WalRawItem: + return "WalRawItem"sv; + case WalShallowItem: + return "WalShallowItem"sv; + case WalTagsMatcher: + return "WalTagsMatcher"sv; + case WalResetLocalWal: + return "WalResetLocalWal"sv; } + return ""sv; } WrSerializer &WALRecord::Dump(WrSerializer &ser, const std::function &cjsonViewer) const { @@ -156,10 +172,13 @@ WrSerializer &WALRecord::Dump(WrSerializer &ser, const std::function &cjsonViewer) const { @@ -191,8 +210,10 @@ void WALRecord::GetJSON(JsonBuilder &jb, const std::function(reinterpret_cast(data.data()), data.size())) {} diff --git a/cpp_src/replicator/walrecord.h b/cpp_src/replicator/walrecord.h index 1b6a0e9e0..bb25452eb 100644 --- a/cpp_src/replicator/walrecord.h +++ b/cpp_src/replicator/walrecord.h @@ -30,6 +30,10 @@ enum WALRecType { WalForceSync = 14, WalSetSchema = 15, WalWALSync = 16, + WalTagsMatcher = 17, + WalResetLocalWal = 18, + WalRawItem = 19, + WalShallowItem = 20, }; class WrSerializer; @@ -57,8 +61,8 @@ struct WALRecord { explicit WALRecord(WALRecType _type, std::string_view key, std::string_view value) : type(_type), putMeta{key, value} {} explicit WALRecord(WALRecType _type, std::string_view cjson, int tmVersion, int modifyMode, bool inTx = false) : type(_type), itemModify{cjson, tmVersion, modifyMode}, inTransaction(inTx) {} - WrSerializer &Dump(WrSerializer &ser, const std::function& cjsonViewer) const; - void GetJSON(JsonBuilder &jb, const std::function& cjsonViewer) const; + WrSerializer &Dump(WrSerializer &ser, const std::function &cjsonViewer) const; + void GetJSON(JsonBuilder &jb, const std::function &cjsonViewer) const; void Pack(WrSerializer &ser) const; SharedWALRecord GetShared(int64_t lsn, int64_t upstreamLSN, std::string_view nsName) const; @@ -75,6 +79,10 @@ struct WALRecord { std::string_view key; std::string_view value; } putMeta; + struct { + IdType id; + std::string_view itemCJson; + } rawItem; }; bool inTransaction = false; mutable SharedWALRecord shared_; diff --git a/cpp_src/replicator/walselecter.cc b/cpp_src/replicator/walselecter.cc index 23da38c5c..12eec12d5 100644 --- a/cpp_src/replicator/walselecter.cc +++ b/cpp_src/replicator/walselecter.cc @@ -106,6 +106,10 @@ void WALSelecter::operator()(QueryResults &result, SelectCtx ¶ms) { case WalNamespaceRename: case WalForceSync: case WalWALSync: + case WalRawItem: + case WalShallowItem: + case WalTagsMatcher: + case WalResetLocalWal: std::abort(); } } diff --git a/cpp_src/server/contrib/server.yml b/cpp_src/server/contrib/server.yml index 2df0c33cd..bdc4fe6bc 100644 --- a/cpp_src/server/contrib/server.yml +++ b/cpp_src/server/contrib/server.yml @@ -2738,8 +2738,8 @@ definitions: type: integer default: 20000 minimum: 0 - maximum: 65535 - description: "Maximum documents count which will be processed in merge query results. Increasing this value may refine ranking of queries with high frequency words, but will decrease search speed" + maximum: 65000 + description: "Maximum documents count which will be processed in merge query results. Increasing this value may refine ranking of queries with high frequency words, but will decrease search speed" extra_word_symbols: type: string default: "-/+" @@ -3984,7 +3984,14 @@ definitions: description: "Enable network traffic compression" cluster_id: type: integer + default: 2 description: "Cluser ID - must be same for client and for master" + server_id: + type: integer + default: 0 + minimun: 0 + maximum: 999 + description: "Node identifier. Should be unique for each node in the replicated cluster (non-unique IDs are also allowed, but may lead to the inconsistency in some cases" force_sync_on_logic_error: type: boolean description: "force resync on logic error conditions" diff --git a/cpp_src/server/httpserver.cc b/cpp_src/server/httpserver.cc index 65bd6dc2f..27e2adc13 100644 --- a/cpp_src/server/httpserver.cc +++ b/cpp_src/server/httpserver.cc @@ -226,11 +226,9 @@ int HTTPServer::GetDatabases(http::Context &ctx) { } if (sortDirection) { - std::sort(dbs.begin(), dbs.end(), [sortDirection](const std::string &lhs, const std::string &rhs) { - if (sortDirection > 0) - return collateCompare(lhs, rhs, SortingPrioritiesTable()) < 0; - else - return collateCompare(lhs, rhs, SortingPrioritiesTable()) > 0; + boost::sort::pdqsort(dbs.begin(), dbs.end(), [sortDirection](const std::string &lhs, const std::string &rhs) { + return (sortDirection > 0) ? (collateCompare(lhs, rhs, SortingPrioritiesTable()) < 0) + : (collateCompare(lhs, rhs, SortingPrioritiesTable()) > 0); }); } @@ -320,11 +318,9 @@ int HTTPServer::GetNamespaces(http::Context &ctx) { } if (sortDirection) { - std::sort(nsDefs.begin(), nsDefs.end(), [sortDirection](const NamespaceDef &lhs, const NamespaceDef &rhs) { - if (sortDirection > 0) - return collateCompare(lhs.name, rhs.name, SortingPrioritiesTable()) < 0; - else - return collateCompare(lhs.name, rhs.name, SortingPrioritiesTable()) > 0; + boost::sort::pdqsort(nsDefs.begin(), nsDefs.end(), [sortDirection](const NamespaceDef &lhs, const NamespaceDef &rhs) { + return (sortDirection > 0) ? (collateCompare(lhs.name, rhs.name, SortingPrioritiesTable()) < 0) + : (collateCompare(lhs.name, rhs.name, SortingPrioritiesTable()) > 0); }); } @@ -543,9 +539,9 @@ int HTTPServer::GetMetaList(http::Context &ctx) { return jsonStatus(ctx, http::HttpStatus(err)); } if (sortDirection == Asc) { - std::sort(keys.begin(), keys.end()); + boost::sort::pdqsort(keys.begin(), keys.end()); } else if (sortDirection == Desc) { - std::sort(keys.begin(), keys.end(), std::greater()); + boost::sort::pdqsort(keys.begin(), keys.end(), std::greater()); } auto keysIt = keys.begin(); auto keysEnd = keys.end(); diff --git a/cpp_src/server/rpcserver.cc b/cpp_src/server/rpcserver.cc index c729cb065..d508c7dcc 100644 --- a/cpp_src/server/rpcserver.cc +++ b/cpp_src/server/rpcserver.cc @@ -175,8 +175,10 @@ void RPCServer::OnClose(cproto::Context &ctx, const Error &err) { if (qrId.main >= 0) { try { qrWatcher_.FreeQueryResults(qrId, false); + // NOLINTBEGIN(bugprone-empty-catch) } catch (...) { } + // NOLINTEND(bugprone-empty-catch) } } } @@ -534,6 +536,8 @@ Error RPCServer::ModifyItem(cproto::Context &ctx, p_string ns, int format, p_str case ModeDelete: err = db.WithTimeout(execTimeout).Delete(ns, item, qres); break; + default: + return Error(errParams, "Unexpected ItemModifyMode: %d", mode); } if (!err.ok()) return err; } else { @@ -550,6 +554,8 @@ Error RPCServer::ModifyItem(cproto::Context &ctx, p_string ns, int format, p_str case ModeDelete: err = db.WithTimeout(execTimeout).Delete(ns, item); break; + default: + return Error(errParams, "Unexpected ItemModifyMode: %d", mode); } if (!err.ok()) return err; qres.AddItem(item); @@ -659,8 +665,10 @@ Error RPCServer::sendResults(cproto::Context &ctx, QueryResults &qres, RPCQrId i freeQueryResults(ctx, id); id.main = -1; id.uid = RPCQrWatcher::kUninitialized; + // NOLINTBEGIN(bugprone-empty-catch) } catch (...) { } + // NOLINTEND(bugprone-empty-catch) } return err; } diff --git a/cpp_src/tools/filecontentwatcher.h b/cpp_src/tools/filecontentwatcher.h index 13ea5a555..6297fc936 100644 --- a/cpp_src/tools/filecontentwatcher.h +++ b/cpp_src/tools/filecontentwatcher.h @@ -11,8 +11,7 @@ class FileContetWatcher { void SetFilepath(std::string filepath, bool enable = false) noexcept { assertrx(!hasFilepath_.load(std::memory_order_acquire)); filepath_ = std::move(filepath); - auto stat = fs::StatTime(filepath_); - lastReplConfMTime_.store(stat.mtime, std::memory_order_relaxed); + lastReplConfMTime_.store(fs::StatTime(filepath_).mtime, std::memory_order_relaxed); hasFilepath_.store(true, std::memory_order_release); if (enable) { isEnabled_.store(true, std::memory_order_release); @@ -31,18 +30,21 @@ class FileContetWatcher { if (!isEnabled_.load(std::memory_order_acquire) || !hasFilepath_.load(std::memory_order_acquire)) { return false; } - auto stat = fs::StatTime(filepath_); - if (stat.mtime > 0) { - auto lastReplConfMTime = lastReplConfMTime_.load(std::memory_order_acquire); - if (lastReplConfMTime != stat.mtime) { - if (lastReplConfMTime_.compare_exchange_strong(lastReplConfMTime, stat.mtime, std::memory_order_acq_rel)) { - auto res = fs::ReadFile(filepath_, content); - if (res < 0) { - content.clear(); - } - std::lock_guard lck(mtx_); - if (content != expectedContent_) { - return true; + auto mtime = fs::StatTime(filepath_).mtime; + if (mtime > 0) { + if (lastReplConfMTime_.load(std::memory_order_acquire) != mtime) { + std::lock_guard lck(mtx_); + mtime = fs::StatTime(filepath_).mtime; + if (mtime > 0) { + if (lastReplConfMTime_.load(std::memory_order_relaxed) != mtime) { + lastReplConfMTime_.store(mtime, std::memory_order_release); + auto res = fs::ReadFile(filepath_, content); + if (res < 0) { + content.clear(); + } + if (content != expectedContent_) { + return true; + } } } } @@ -53,18 +55,18 @@ class FileContetWatcher { template Error RewriteFile(std::string content, PredicatT hasSameContent) { if (!isEnabled_.load(std::memory_order_acquire) || !hasFilepath_.load(std::memory_order_acquire)) { - return errOK; + return Error(); } - std::string tmpPath = filepath_ + ".tmp"; std::string curContent; auto res = fs::ReadFile(filepath_, curContent); if (res < 0) { - return errOK; + return Error(); } if (hasSameContent(curContent)) { - return errOK; + return Error(); } + const std::string tmpPath = filepath_ + ".tmp"; std::lock_guard lck(mtx_); res = fs::WriteFile(tmpPath, content); if (res < 0 || static_cast(res) != content.size()) { @@ -75,7 +77,11 @@ class FileContetWatcher { return Error(errParams, "Unable to rename tmp file from [%s] to [%s]. Reason: %s", tmpPath, filepath_, strerror(errno)); } expectedContent_ = std::move(content); - return errOK; + auto stat = fs::StatTime(filepath_); + if (stat.mtime > 0) { + lastReplConfMTime_.store(stat.mtime, std::memory_order_release); + } + return Error(); } private: diff --git a/cpp_src/tools/fsops.cc b/cpp_src/tools/fsops.cc index e7d8db748..5fbfcae71 100644 --- a/cpp_src/tools/fsops.cc +++ b/cpp_src/tools/fsops.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include "errors.h" #include "tools/oscompat.h" @@ -177,7 +178,16 @@ std::string GetCwd() { return std::string(getcwd(buff, FILENAME_MAX)); } +static std::string tmpDir; +static std::mutex tmpDirMtx; + std::string GetTempDir() { + { + std::lock_guard lck(tmpDirMtx); + if (!tmpDir.empty()) { + return tmpDir; + } + } #ifdef _WIN32 char tmpBuf[512]; *tmpBuf = 0; @@ -190,6 +200,11 @@ std::string GetTempDir() { #endif } +void SetTempDir(std::string &&dir) noexcept { + std::lock_guard lck(tmpDirMtx); + tmpDir = std::move(dir); +} + std::string GetHomeDir() { const char *homeDir = getenv("HOME"); if (homeDir && *homeDir) return homeDir; @@ -355,5 +370,6 @@ std::string GetRelativePath(const std::string &path, unsigned maxUp) { rpath.append(path.begin() + same, path.end()); return rpath; } + } // namespace fs } // namespace reindexer diff --git a/cpp_src/tools/fsops.h b/cpp_src/tools/fsops.h index 46d56db4c..015ef0a66 100644 --- a/cpp_src/tools/fsops.h +++ b/cpp_src/tools/fsops.h @@ -37,6 +37,7 @@ TimeStats StatTime(const std::string &path); std::string GetCwd(); std::string GetDirPath(const std::string &path); std::string GetTempDir(); +void SetTempDir(std::string &&dir) noexcept; std::string GetHomeDir(); std::string GetRelativePath(const std::string &path, unsigned maxUp = 1024); inline int Rename(const std::string &from, const std::string &to) { return rename(from.c_str(), to.c_str()); } diff --git a/cpp_src/tools/jsonstring.h b/cpp_src/tools/jsonstring.h index 34c6eeb56..36601af0a 100644 --- a/cpp_src/tools/jsonstring.h +++ b/cpp_src/tools/jsonstring.h @@ -71,12 +71,15 @@ inline void encode(uint8_t *p, uint64_t l, std::vector> p[-3] = (uptr >> 8) & 0xFF; p[-4] = (uptr >> 16) & 0xFF; p[-5] = (uptr >> 24) & 0xFF; - if constexpr (sizeof(uintptr_t) == 8) { - p[-6] = (uptr >> 32) & 0xFF; - p[-7] = (uptr >> 40) & 0xFF; - p[-8] = (uptr >> 48) & 0xFF; - p[-9] = (uptr >> 56) & 0xFF; - } +#if UINTPTR_MAX == 0xFFFFFFFF +#elif UINTPTR_MAX == 0xFFFFFFFFFFFFFFFF + p[-6] = (uptr >> 32) & 0xFF; + p[-7] = (uptr >> 40) & 0xFF; + p[-8] = (uptr >> 48) & 0xFF; + p[-9] = (uptr >> 56) & 0xFF; +#else + static_assert(false, "Unexpected uintptr_t size"); +#endif } else { // Put length p[0] = l & 0xFF; diff --git a/cpp_src/tools/serializer.cc b/cpp_src/tools/serializer.cc index ae4880b77..2b7dd1a87 100644 --- a/cpp_src/tools/serializer.cc +++ b/cpp_src/tools/serializer.cc @@ -3,7 +3,6 @@ #include "core/keyvalue/p_string.h" #include "estl/span.h" #include "tools/errors.h" -#include "vendor/double-conversion/double-conversion.h" #include "vendor/itoa/itoa.h" namespace reindexer { @@ -81,19 +80,6 @@ void WrSerializer::VStringHelper::End() { } } -WrSerializer &WrSerializer::operator<<(double v) { - grow(32); - double_conversion::StringBuilder builder(reinterpret_cast(buf_ + len_), 32); - int flags = - double_conversion::DoubleToStringConverter::UNIQUE_ZERO | double_conversion::DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN; - double_conversion::DoubleToStringConverter dc(flags, NULL, NULL, 'e', -6, 21, 0, 0); - - dc.ToShortest(v, &builder); - len_ += builder.position(); - - return *this; -} - void WrSerializer::PrintJsonString(std::string_view str, PrintJsonStringMode mode) { const char *s = str.data(); size_t l = str.size(); diff --git a/cpp_src/tools/serializer.h b/cpp_src/tools/serializer.h index 1765279db..b206eb37e 100644 --- a/cpp_src/tools/serializer.h +++ b/cpp_src/tools/serializer.h @@ -7,6 +7,7 @@ #include "estl/chunk.h" #include "estl/one_of.h" #include "estl/span.h" +#include "tools/stringstools.h" #include "tools/varint.h" char *i32toa(int32_t value, char *buffer); @@ -311,6 +312,10 @@ class WrSerializer { memcpy(&buf_[len_], &v, sizeof(v)); len_ += sizeof(v); } + void PutDoubleStrNoTrailing(double v) { + grow(32); + len_ += double_to_str_no_trailing(v, reinterpret_cast(buf_ + len_), 32); + } template ::value>::type * = nullptr> WrSerializer &operator<<(T k) { @@ -345,7 +350,11 @@ class WrSerializer { Write(v ? "true"sv : "false"sv); return *this; } - WrSerializer &operator<<(double v); + WrSerializer &operator<<(double v) { + grow(32); + len_ += double_to_str(v, reinterpret_cast(buf_ + len_), 32); + return *this; + } WrSerializer &operator<<(Uuid uuid) { grow(Uuid::kStrFormLen + 2); buf_[len_] = '\''; diff --git a/cpp_src/tools/stringstools.cc b/cpp_src/tools/stringstools.cc index 1517c379b..dbf99ecf1 100644 --- a/cpp_src/tools/stringstools.cc +++ b/cpp_src/tools/stringstools.cc @@ -3,19 +3,126 @@ #include #include #include -#include #include #include "atoi/atoi.h" +#include "estl/fast_hash_map.h" #include "itoa/itoa.h" #include "tools/assertrx.h" #include "tools/customlocal.h" #include "tools/randomgenerator.h" #include "tools/stringstools.h" #include "utf8cpp/utf8.h" +#include "vendor/double-conversion/double-conversion.h" namespace reindexer { +namespace stringtools_impl { + +static int double_to_str(double v, char *buf, int capacity, int flags) { + double_conversion::StringBuilder builder(buf, capacity); + double_conversion::DoubleToStringConverter dc(flags, NULL, NULL, 'e', -6, 21, 0, 0); + + if (!dc.ToShortest(v, &builder)) { + // NaN/Inf are not allowed here + throw Error(errParams, "Unable to convert '%f' to string", v); + } + return builder.position(); +} + +static std::pair word2Pos(std::string_view str, int wordPos, int endPos, const std::string &extraWordSymbols) { + auto wordStartIt = str.begin(); + auto wordEndIt = str.begin(); + auto it = str.begin(); + assertrx(endPos > wordPos); + int numWords = endPos - (wordPos + 1); + for (; it != str.end();) { + auto ch = utf8::unchecked::next(it); + + while (it != str.end() && extraWordSymbols.find(ch) == std::string::npos && !IsAlpha(ch) && !IsDigit(ch)) { + wordStartIt = it; + ch = utf8::unchecked::next(it); + } + + while (IsAlpha(ch) || IsDigit(ch) || extraWordSymbols.find(ch) != std::string::npos) { + wordEndIt = it; + if (it == str.end()) break; + ch = utf8::unchecked::next(it); + } + + if (wordStartIt != it) { + if (!wordPos) + break; + else { + wordPos--; + wordStartIt = it; + } + } + } + + for (; numWords != 0 && it != str.end(); numWords--) { + auto ch = utf8::unchecked::next(it); + + while (it != str.end() && !IsAlpha(ch) && !IsDigit(ch)) { + ch = utf8::unchecked::next(it); + } + + while (IsAlpha(ch) || IsDigit(ch) || extraWordSymbols.find(ch) != std::string::npos) { + wordEndIt = it; + if (it == str.end()) break; + ch = utf8::unchecked::next(it); + } + } + + return {int(std::distance(str.begin(), wordStartIt)), int(std::distance(str.begin(), wordEndIt))}; +} + +static std::string_view urldecode2(char *buf, std::string_view str) { + char a, b; + const char *src = str.data(); + char *dst = buf; + + for (size_t l = 0; l < str.length(); l++) { + if (l + 2 < str.length() && (*src == '%') && ((a = src[1]) && (b = src[2])) && (isxdigit(a) && isxdigit(b))) { + if (a >= 'a') a -= 'a' - 'A'; + if (a >= 'A') + a -= ('A' - 10); + else + a -= '0'; + if (b >= 'a') b -= 'a' - 'A'; + if (b >= 'A') + b -= ('A' - 10); + else + b -= '0'; + *dst++ = 16 * a + b; + src += 3; + l += 2; + } else if (*src == '+') { + *dst++ = ' '; + src++; + } else { + *dst++ = *src++; + } + } + *dst = '\0'; + return std::string_view(buf, dst - buf); +} + +// Sat Jul 15 14 : 18 : 56 2017 GMT + +static const char *kDaysOfWeek[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; +static const char *kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + +inline static char *strappend(char *dst, const char *src) noexcept { + while (*src) *dst++ = *src++; + return dst; +} + +static fast_hash_map kStrictModes = { + {"", StrictModeNotSet}, {"none", StrictModeNone}, {"names", StrictModeNames}, {"indexes", StrictModeIndexes}}; + +} // namespace stringtools_impl + std::string toLower(std::string_view src) { std::string ret; ret.reserve(src.size()); @@ -238,63 +345,13 @@ Pos wordToByteAndCharPos(std::string_view str, int wordPosition, const std::stri template WordPositionEx wordToByteAndCharPos(std::string_view str, int wordPosition, const std::string &extraWordSymbols); template WordPosition wordToByteAndCharPos(std::string_view str, int wordPosition, const std::string &extraWordSymbols); -static std::pair word2Pos(std::string_view str, int wordPos, int endPos, const std::string &extraWordSymbols) { - auto wordStartIt = str.begin(); - auto wordEndIt = str.begin(); - auto it = str.begin(); - assertrx(endPos > wordPos); - int numWords = endPos - (wordPos + 1); - for (; it != str.end();) { - auto ch = utf8::unchecked::next(it); - - while (it != str.end() && extraWordSymbols.find(ch) == std::string::npos && !IsAlpha(ch) && !IsDigit(ch)) { - wordStartIt = it; - ch = utf8::unchecked::next(it); - } - - while (IsAlpha(ch) || IsDigit(ch) || extraWordSymbols.find(ch) != std::string::npos) { - wordEndIt = it; - if (it == str.end()) break; - ch = utf8::unchecked::next(it); - } - - if (wordStartIt != it) { - if (!wordPos) - break; - else { - wordPos--; - wordStartIt = it; - } - } - } - - for (; numWords != 0 && it != str.end(); numWords--) { - auto ch = utf8::unchecked::next(it); - - while (it != str.end() && !IsAlpha(ch) && !IsDigit(ch)) { - ch = utf8::unchecked::next(it); - } - - while (IsAlpha(ch) || IsDigit(ch) || extraWordSymbols.find(ch) != std::string::npos) { - wordEndIt = it; - if (it == str.end()) break; - ch = utf8::unchecked::next(it); - } - } - - return {int(std::distance(str.begin(), wordStartIt)), int(std::distance(str.begin(), wordEndIt))}; -} - -Word2PosHelper::Word2PosHelper(std::string_view data, const std::string &extraWordSymbols) - : data_(data), lastWordPos_(0), lastOffset_(0), extraWordSymbols_(extraWordSymbols) {} - std::pair Word2PosHelper::convert(int wordPos, int endPos) { if (wordPos < lastWordPos_) { lastWordPos_ = 0; lastOffset_ = 0; } - auto ret = word2Pos(data_.substr(lastOffset_), wordPos - lastWordPos_, endPos - lastWordPos_, extraWordSymbols_); + auto ret = stringtools_impl::word2Pos(data_.substr(lastOffset_), wordPos - lastWordPos_, endPos - lastWordPos_, extraWordSymbols_); ret.first += lastOffset_; ret.second += lastOffset_; lastOffset_ = ret.first; @@ -453,62 +510,23 @@ int collateCompare(std::string_view lhs, std::string_view rhs, cons return res ? res : ((l1 < l2) ? -1 : (l1 > l2) ? 1 : 0); } -static std::string_view urldecode2(char *buf, std::string_view str) { - char a, b; - const char *src = str.data(); - char *dst = buf; - - for (size_t l = 0; l < str.length(); l++) { - if (l + 2 < str.length() && (*src == '%') && ((a = src[1]) && (b = src[2])) && (isxdigit(a) && isxdigit(b))) { - if (a >= 'a') a -= 'a' - 'A'; - if (a >= 'A') - a -= ('A' - 10); - else - a -= '0'; - if (b >= 'a') b -= 'a' - 'A'; - if (b >= 'A') - b -= ('A' - 10); - else - b -= '0'; - *dst++ = 16 * a + b; - src += 3; - l += 2; - } else if (*src == '+') { - *dst++ = ' '; - src++; - } else { - *dst++ = *src++; - } - } - *dst = '\0'; - return std::string_view(buf, dst - buf); -} - std::string urldecode2(std::string_view str) { std::string ret(str.length(), ' '); - std::string_view sret = urldecode2(&ret[0], str); + std::string_view sret = stringtools_impl::urldecode2(&ret[0], str); ret.resize(sret.size()); return ret; } -// Sat Jul 15 14 : 18 : 56 2017 GMT - -static const char *daysOfWeek[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; -static const char *months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; - -inline static char *strappend(char *dst, const char *src) noexcept { - while (*src) *dst++ = *src++; - return dst; -} - int fast_strftime(char *buf, const tm *tm) { char *d = buf; - if (unsigned(tm->tm_wday) < sizeof(daysOfWeek) / sizeof daysOfWeek[0]) d = strappend(d, daysOfWeek[tm->tm_wday]); - d = strappend(d, ", "); + if (unsigned(tm->tm_wday) < sizeof(stringtools_impl::kDaysOfWeek) / sizeof stringtools_impl::kDaysOfWeek[0]) + d = stringtools_impl::strappend(d, stringtools_impl::kDaysOfWeek[tm->tm_wday]); + d = stringtools_impl::strappend(d, ", "); d = i32toa(tm->tm_mday, d); *d++ = ' '; - if (unsigned(tm->tm_mon) < sizeof(months) / sizeof months[0]) d = strappend(d, months[tm->tm_mon]); + if (unsigned(tm->tm_mon) < sizeof(stringtools_impl::kMonths) / sizeof stringtools_impl::kMonths[0]) + d = stringtools_impl::strappend(d, stringtools_impl::kMonths[tm->tm_mon]); *d++ = ' '; d = i32toa(tm->tm_year + 1900, d); *d++ = ' '; @@ -517,7 +535,7 @@ int fast_strftime(char *buf, const tm *tm) { d = i32toa(tm->tm_min, d); *d++ = ':'; d = i32toa(tm->tm_sec, d); - d = strappend(d, " GMT"); + d = stringtools_impl::strappend(d, " GMT"); *d = 0; return d - buf; } @@ -546,8 +564,8 @@ bool validateUserNsName(std::string_view name) noexcept { return true; } -LogLevel logLevelFromString(const std::string &strLogLevel) { - static std::unordered_map levels = { +LogLevel logLevelFromString(std::string_view strLogLevel) { + static fast_hash_map levels = { {"none", LogNone}, {"warning", LogWarning}, {"error", LogError}, {"info", LogInfo}, {"trace", LogTrace}}; auto configLevelIt = levels.find(strLogLevel); @@ -557,25 +575,21 @@ LogLevel logLevelFromString(const std::string &strLogLevel) { return LogNone; } -static std::unordered_map strictModes = { - {"", StrictModeNotSet}, {"none", StrictModeNone}, {"names", StrictModeNames}, {"indexes", StrictModeIndexes}}; - -StrictMode strictModeFromString(const std::string &strStrictMode) { - auto configModeIt = strictModes.find(strStrictMode); - if (configModeIt != strictModes.end()) { +StrictMode strictModeFromString(std::string_view strStrictMode) { + auto configModeIt = stringtools_impl::kStrictModes.find(strStrictMode); + if (configModeIt != stringtools_impl::kStrictModes.end()) { return configModeIt->second; } return StrictModeNotSet; } std::string_view strictModeToString(StrictMode mode) { - for (auto &it : strictModes) { + for (auto &it : stringtools_impl::kStrictModes) { if (it.second == mode) { return it.first; } } - static std::string_view empty{""}; - return empty; + return std::string_view(); } bool isPrintable(std::string_view str) noexcept { @@ -635,6 +649,26 @@ int64_t stoll(std::string_view sl) { return ret; } +int double_to_str(double v, char *buf, int capacity) { + const int flags = double_conversion::DoubleToStringConverter::UNIQUE_ZERO | + double_conversion::DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN | + double_conversion::DoubleToStringConverter::EMIT_TRAILING_DECIMAL_POINT | + double_conversion::DoubleToStringConverter::EMIT_TRAILING_ZERO_AFTER_POINT; + return stringtools_impl::double_to_str(v, buf, capacity, flags); +} +int double_to_str_no_trailing(double v, char *buf, int capacity) { + const int flags = + double_conversion::DoubleToStringConverter::UNIQUE_ZERO | double_conversion::DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN; + return stringtools_impl::double_to_str(v, buf, capacity, flags); +} +std::string double_to_str(double v) { + std::string res; + res.resize(32); + auto len = double_to_str(v, res.data(), res.size()); + res.resize(len); + return res; +} + std::string randStringAlph(size_t len) { constexpr std::string_view symbols = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; std::string result; diff --git a/cpp_src/tools/stringstools.h b/cpp_src/tools/stringstools.h index ed0e1a41b..97c451577 100644 --- a/cpp_src/tools/stringstools.h +++ b/cpp_src/tools/stringstools.h @@ -63,7 +63,8 @@ int getUTF8StringCharactersCount(std::string_view str) noexcept; class Word2PosHelper { public: - Word2PosHelper(std::string_view data, const std::string& extraWordSymbols); + Word2PosHelper(std::string_view data, const std::string& extraWordSymbols) noexcept + : data_(data), lastWordPos_(0), lastOffset_(0), extraWordSymbols_(extraWordSymbols) {} std::pair convert(int wordPos, int endPos); protected: @@ -145,12 +146,15 @@ std::string urldecode2(std::string_view str); int stoi(std::string_view sl); std::optional try_stoi(std::string_view sl); int64_t stoll(std::string_view sl); +int double_to_str(double v, char* buf, int capacity); +int double_to_str_no_trailing(double v, char* buf, int capacity); +std::string double_to_str(double v); bool validateObjectName(std::string_view name, bool allowSpecialChars) noexcept; bool validateUserNsName(std::string_view name) noexcept; RX_ALWAYS_INLINE bool isSystemNamespaceNameFast(std::string_view name) noexcept { return !name.empty() && name[0] == '#'; } -LogLevel logLevelFromString(const std::string& strLogLevel); -StrictMode strictModeFromString(const std::string& strStrictMode); +LogLevel logLevelFromString(std::string_view strLogLevel); +StrictMode strictModeFromString(std::string_view strStrictMode); std::string_view strictModeToString(StrictMode mode); inline bool iequals(std::string_view lhs, std::string_view rhs) noexcept { diff --git a/cpp_src/vendor/gason/gason.h b/cpp_src/vendor/gason/gason.h index 6e227f7a8..76c218d37 100644 --- a/cpp_src/vendor/gason/gason.h +++ b/cpp_src/vendor/gason/gason.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "estl/span.h" #include "tools/jsonstring.h" diff --git a/cpp_src/vendor/yaml-cpp/binary.cpp b/cpp_src/vendor/yaml-cpp/binary.cpp index 55fc1abd2..4e75b6628 100644 --- a/cpp_src/vendor/yaml-cpp/binary.cpp +++ b/cpp_src/vendor/yaml-cpp/binary.cpp @@ -37,6 +37,8 @@ std::string EncodeBase64(const unsigned char *data, std::size_t size) { *out++ = encoding[((data[1] & 0xf) << 2)]; *out++ = PAD; break; + default: + std::abort(); } ret.resize(out - &ret[0]); diff --git a/cpp_src/vendor/yaml-cpp/emitterutils.cpp b/cpp_src/vendor/yaml-cpp/emitterutils.cpp index f92106c43..4f9543d58 100644 --- a/cpp_src/vendor/yaml-cpp/emitterutils.cpp +++ b/cpp_src/vendor/yaml-cpp/emitterutils.cpp @@ -32,6 +32,7 @@ bool IsAnchorChar(int ch) { // test for ns-anchor-char return false; case 0x85: return true; + default:; } if (ch < 0x20) { diff --git a/cpp_src/vendor/yaml-cpp/exp.cpp b/cpp_src/vendor/yaml-cpp/exp.cpp index 976ef13f9..814e68894 100644 --- a/cpp_src/vendor/yaml-cpp/exp.cpp +++ b/cpp_src/vendor/yaml-cpp/exp.cpp @@ -121,6 +121,7 @@ std::string Escape(Stream& in) { return Escape(in, 4); case 'U': return Escape(in, 8); + default:; } std::stringstream msg; diff --git a/cpp_src/vendor/yaml-cpp/stream.cpp b/cpp_src/vendor/yaml-cpp/stream.cpp index ed58ef8cb..c387beb81 100644 --- a/cpp_src/vendor/yaml-cpp/stream.cpp +++ b/cpp_src/vendor/yaml-cpp/stream.cpp @@ -107,6 +107,7 @@ inline UtfIntroCharType IntroCharTypeOf(std::istream::int_type ch) { return uictFE; case 0xFF: return uictFF; + default:; } if ((ch > 0) && (ch < 0xFF)) { diff --git a/describer.go b/describer.go index ed494bed8..b356b3cba 100644 --- a/describer.go +++ b/describer.go @@ -85,7 +85,7 @@ type CacheMemStat struct { type LsnT struct { // Operation counter Counter int64 `json:"counter"` - // Node identifyer + // Node identifier ServerId int `json:"server_id"` } diff --git a/fulltext.md b/fulltext.md index e741225a2..672ab7e52 100644 --- a/fulltext.md +++ b/fulltext.md @@ -378,10 +378,10 @@ FtTyposDetailedConfig: config for more precise typos algorithm tuning. | | Parameter name | Type | Description | Default value | |---|:----------------------------:|:--------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-------------:| -| | MaxTypoDistance | int | Maximum distance between symbols in initial and target words to perform substitution. Check [typos handling](#typos-handling-details) section for detailed description. | 0 | -| | MaxSymbolPermutationDistance | int | aximum distance between same symbols in initial and target words to perform substitution (to handle cases, when two symbolws were switched with each other). Check [typos handling](#typos-handling-details) section for detailed description. | 1 | -| | MaxMissingLetters | int | Maximum number of symbols, which may be removed from the initial term to transform it into the result word. Check [typos handling](#typos-handling-details) section for detailed description. | 2 | -| | MaxExtraLetters | int | Maximum number of symbols, which may be added to the initial term to transform it into the result word. Check [typos handling](#typos-handling-details) section for detailed description. | 2 | +| | MaxTypoDistance | int | Maximum distance between symbols in initial and target words to perform substitution. Check [typos handling](#typos-handling-details) section for detailed description. | 0 | +| | MaxSymbolPermutationDistance | int | aximum distance between same symbols in initial and target words to perform substitution (to handle cases, when two symbolws were switched with each other). Check [typos handling](#typos-handling-details) section for detailed description. | 1 | +| | MaxMissingLetters | int | Maximum number of symbols, which may be removed from the initial term to transform it into the result word. Check [typos handling](#typos-handling-details) section for detailed description. | 2 | +| | MaxExtraLetters | int | Maximum number of symbols, which may be added to the initial term to transform it into the result word. Check [typos handling](#typos-handling-details) section for detailed description. | 2 | ### Base ranking config diff --git a/query.go b/query.go index 804fdbafe..cd65d7cbe 100644 --- a/query.go +++ b/query.go @@ -731,9 +731,6 @@ func (q *Query) ExecCtx(ctx context.Context) *Iterator { if q.executed { q.panicTrace("Exec call on already executed query. You should create new Query") } - // if q.tx != nil { - // panic(errors.New("For tx query only Update or Delete operations are supported")) - // } q.executed = true @@ -756,9 +753,6 @@ func (q *Query) ExecToJsonCtx(ctx context.Context, jsonRoots ...string) *JSONIte if q.executed { q.panicTrace("Exec call on already executed query. You should create new Query") } - // if q.tx != nil { - // panic(errors.New("For tx query only Update or Delete operations are supported")) - // } q.executed = true @@ -1090,7 +1084,17 @@ func (q *Query) LeftJoin(q2 *Query, field string) *Query { } // JoinHandler registers join handler that will be called when join, registered on `field` value, finds a match +// Handler will be always set to the main query func (q *Query) JoinHandler(field string, handler JoinHandler) *Query { + if q.root != nil { + // Joined queries can not have JoinHandlers themselfs. Routing this call to the root query if current query is joined + for _, jq := range q.root.joinQueries { + if q == jq { + q.root.JoinHandler(field, handler) + return q + } + } + } index := -1 for i := range q.joinToFields { if strings.EqualFold(q.joinToFields[i], field) { diff --git a/test/builtinserver_test.go b/test/builtinserver_test.go index 978e23c6d..2b96c59b1 100644 --- a/test/builtinserver_test.go +++ b/test/builtinserver_test.go @@ -19,7 +19,7 @@ func TestBuiltinServer(t *testing.T) { cfg1 := config.DefaultServerConfig() cfg1.Net.HTTPAddr = "0:29088" cfg1.Net.RPCAddr = "0:26534" - cfg1.Storage.Path = "/tmp/rx_builtinserver_test1" + cfg1.Storage.Path = "/tmp/reindex_builtinserver_test1" os.RemoveAll(cfg1.Storage.Path) rx1 := reindexer.NewReindex("builtinserver://xxx", reindexer.WithServerConfig(time.Second*100, cfg1)) @@ -30,7 +30,7 @@ func TestBuiltinServer(t *testing.T) { cfg2 := config.DefaultServerConfig() cfg2.Net.HTTPAddr = "0:29089" cfg2.Net.RPCAddr = "0:26535" - cfg2.Storage.Path = "/tmp/rx_builtinserver_test2" + cfg2.Storage.Path = "/tmp/reindex_builtinserver_test2" os.RemoveAll(cfg2.Storage.Path) rx2 := reindexer.NewReindex("builtinserver://xxx", reindexer.WithServerConfig(time.Second*100, cfg2)) @@ -46,7 +46,7 @@ func TestBuiltinServer(t *testing.T) { cfg4 := config.DefaultServerConfig() cfg4.Net.HTTPAddr = "0:29090" cfg4.Net.RPCAddr = "0:26536" - cfg4.Storage.Path = "/tmp/rx_builtinserver_test4" + cfg4.Storage.Path = "/tmp/reindex_builtinserver_test4" cfg4.Net.UnixRPCAddr = "/tmp/reindexer_builtinserver_test.sock" os.RemoveAll(cfg4.Storage.Path) diff --git a/test/compatibility_test/compatibility_test.sh b/test/compatibility_test/compatibility_test.sh index 9300f0e77..31b893ff3 100755 --- a/test/compatibility_test/compatibility_test.sh +++ b/test/compatibility_test/compatibility_test.sh @@ -39,19 +39,19 @@ test_outdated_instance() { echo "====Master: ${master_cmd}" echo "====Slave: ${slave_cmd}" init_storages - ${master_cmd} --db "${master_db_path}" -l0 --serverlog=\"\" --corelog=\"\" --httplog=\"\" --rpclog=\"\" & + ${master_cmd} --db "${master_db_path}" -l0 --serverlog=\"reindexer_master_$3.1.log\" --corelog=\"reindexer_master_$3.1.log\" --httplog=\"\" --rpclog=\"\" & master_pid=$! sleep 4 go run ${script_dir}/filler.go --dsn "${master_dsn}/${db_name}" --offset 0 echo "====Force sync" - ${slave_cmd} --db "${slave_db_path}" -p 9089 -r 6535 -l0 --serverlog=\"\" --corelog=\"\" --httplog=\"\" --rpclog=\"\" & + ${slave_cmd} --db "${slave_db_path}" -p 9089 -r 6535 -l0 --serverlog=\"reindexer_slave_$3.1.log\" --corelog=\"reindexer_slave_$3.1.log\" --httplog=\"\" --rpclog=\"\" & slave_pid=$! sleep 5 kill $slave_pid wait $slave_pid go run ${script_dir}/filler.go --dsn "${master_dsn}/${db_name}" --offset 100 echo "====Sync by WAL" - ${slave_cmd} --db "${slave_db_path}" -p 9089 -r 6535 -l0 --serverlog=\"\" --corelog=\"\" --httplog=\"\" --rpclog=\"\" & + ${slave_cmd} --db "${slave_db_path}" -p 9089 -r 6535 -l0 --serverlog=\"reindexer_slave_$3.2.log\" --corelog=\"reindexer_slave_$3.2.log\" --httplog=\"\" --rpclog=\"\" & slave_pid=$! sleep 5 echo "====Online sync" @@ -71,8 +71,8 @@ echo "====Installing reindexer package====" echo "====URL: ${rpm_url}" yum install -y ${rpm_url} > /dev/null || true echo "====Checking outdated slave====" -test_outdated_instance "build/cpp_src/cmd/reindexer_server/reindexer_server" "reindexer_server" +test_outdated_instance "build/cpp_src/cmd/reindexer_server/reindexer_server" "reindexer_server" "1" echo "====Checking outdated master====" -test_outdated_instance "reindexer_server" "build/cpp_src/cmd/reindexer_server/reindexer_server" +test_outdated_instance "reindexer_server" "build/cpp_src/cmd/reindexer_server/reindexer_server" "2" clear_artifacts diff --git a/test/fields_autogen_test.go b/test/fields_autogen_test.go index 0813cb0af..2f25b6176 100644 --- a/test/fields_autogen_test.go +++ b/test/fields_autogen_test.go @@ -26,6 +26,9 @@ func init() { } func TestAutogen(t *testing.T) { + + currentSerial := 0 + t.Run("field should be updated with current timestamp using NOW() function", func(t *testing.T) { precepts := []string{"updated_time=NOW()"} item := TestItemAutogen{} @@ -51,9 +54,10 @@ func TestAutogen(t *testing.T) { item := TestItemAutogen{} err := DB.Upsert(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 - assert.Equal(t, 1, item.Age) - assert.Equal(t, int64(1), item.Genre) + assert.Equal(t, currentSerial, item.Age) + assert.Equal(t, int64(currentSerial), item.Genre) t.Run("serial field should be increased by 5 after 5 iterations (must be equal 6 after previous test)", func(t *testing.T) { precepts := []string{"genre=SERIAL()", "age=serial()"} @@ -61,70 +65,84 @@ func TestAutogen(t *testing.T) { for i := 0; i < 5; i++ { err := DB.Upsert(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 } - assert.Equal(t, 6, item.Age) - assert.Equal(t, int64(6), item.Genre) + assert.Equal(t, currentSerial, item.Age) + assert.Equal(t, int64(currentSerial), item.Genre) }) }) t.Run("fill on insert, update, upsert", func(t *testing.T) { - precepts := []string{"updated_time=NOW()"} + precepts := []string{"updated_time=NOW()", "age=SERIAL()"} item := TestItemAutogen{ID: rand.Intn(100000000)} _, err := DB.Insert(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 assert.GreaterOrEqual(t, item.UpdatedTime, time.Now().Unix()-1) assert.LessOrEqual(t, item.UpdatedTime, time.Now().Unix()) + assert.Equal(t, currentSerial, item.Age) item = TestItemAutogen{} err = DB.Upsert(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 assert.GreaterOrEqual(t, item.UpdatedTime, time.Now().Unix()-1) assert.LessOrEqual(t, item.UpdatedTime, time.Now().Unix()) + assert.Equal(t, currentSerial, item.Age) item = TestItemAutogen{} _, err = DB.Update(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 assert.GreaterOrEqual(t, item.UpdatedTime, time.Now().Unix()-1) assert.LessOrEqual(t, item.UpdatedTime, time.Now().Unix()) + assert.Equal(t, currentSerial, item.Age) }) - t.Run("fill on upsert not exist item", func(t *testing.T) { - precepts := []string{"updated_time=NOW()"} + t.Run("fill on upsert nonexist item", func(t *testing.T) { + precepts := []string{"updated_time=NOW()", "age=SERIAL()"} item := TestItemAutogen{ID: rand.Intn(100000000)} err := DB.Upsert(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 assert.GreaterOrEqual(t, item.UpdatedTime, time.Now().Unix()-1) assert.LessOrEqual(t, item.UpdatedTime, time.Now().Unix()) + assert.Equal(t, currentSerial, item.Age) }) - t.Run("not fill on update not exist item", func(t *testing.T) { - precepts := []string{"updated_time=NOW()"} + t.Run("doesn't fill on update nonexist item", func(t *testing.T) { + precepts := []string{"updated_time=NOW()", "age=SERIAL()"} item := TestItemAutogen{ID: rand.Intn(100000000)} count, err := DB.Update(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 // remove after 1602 assert.Equal(t, 0, count) assert.Equal(t, int64(0), item.UpdatedTime) + assert.Equal(t, 0, item.Age) }) - t.Run("not fill on insert exist item", func(t *testing.T) { - precepts := []string{"updated_time=NOW()"} + t.Run("doesn't fill on insert exist item", func(t *testing.T) { + precepts := []string{"updated_time=NOW()", "age=SERIAL()"} id := rand.Intn(100000000) item := TestItemAutogen{ID: id} count, err := DB.Insert(ns, &item, precepts...) require.NoError(t, err) + currentSerial += 1 assert.Equal(t, 1, count) assert.GreaterOrEqual(t, item.UpdatedTime, time.Now().Unix()-1) assert.LessOrEqual(t, item.UpdatedTime, time.Now().Unix()) + assert.Equal(t, currentSerial, item.Age) item = TestItemAutogen{ID: id} count, err = DB.Insert(ns, &item, precepts...) require.NoError(t, err) assert.Equal(t, 0, count) assert.Equal(t, int64(0), item.UpdatedTime) + assert.Equal(t, 0, item.Age) }) } diff --git a/test/queries_test.go b/test/queries_test.go index 440217696..6d6dee195 100644 --- a/test/queries_test.go +++ b/test/queries_test.go @@ -589,9 +589,12 @@ func TestSTDistanceWrappers(t *testing.T) { searchPoint := randPoint() distance := randFloat(0, 2) sortPoint := randPoint() - it1, err := DBD.Query(ns).DWithin(field1, searchPoint, distance).SortStPointDistance(field1, sortPoint, false).ExecToJson().FetchAll() + it1, err := DBD.Query(ns).DWithin(field1, searchPoint, distance).SortStPointDistance(field1, sortPoint, false). + ExecToJson().FetchAll() require.NoError(t, err) - it2, err := DBD.Query(ns).DWithin(field1, searchPoint, distance).Sort(fmt.Sprintf("ST_Distance(%s, ST_GeomFromText('point(%f %f)'))", field1, sortPoint[0], sortPoint[1]), false).ExecToJson().FetchAll() + it2, err := DBD.Query(ns).DWithin(field1, searchPoint, distance).Sort(fmt.Sprintf("ST_Distance(%s, ST_GeomFromText('point(%s %s)'))", + field1, strconv.FormatFloat(sortPoint[0], 'f', -1, 64), strconv.FormatFloat(sortPoint[1], 'f', -1, 64)), false). + ExecToJson().FetchAll() require.NoError(t, err) require.Equal(t, string(it1), string(it2)) } diff --git a/test/slave_empty_storage_test.go b/test/slave_empty_storage_test.go index 3c05a0bad..7f4d889ef 100644 --- a/test/slave_empty_storage_test.go +++ b/test/slave_empty_storage_test.go @@ -21,7 +21,7 @@ func TestSlaveEmptyStorage(t *testing.T) { cfgMaster := config.DefaultServerConfig() cfgMaster.Net.HTTPAddr = "0:29088" cfgMaster.Net.RPCAddr = "0:26534" - cfgMaster.Storage.Path = "/tmp/rx_master1" + cfgMaster.Storage.Path = "/tmp/reindex_master1" os.RemoveAll(cfgMaster.Storage.Path) rxMaster := reindexer.NewReindex("builtinserver://xxx", reindexer.WithServerConfig(time.Second*100, cfgMaster)) { @@ -45,7 +45,7 @@ namespaces: []` cfgSlave := config.DefaultServerConfig() cfgSlave.Net.HTTPAddr = "0:29089" cfgSlave.Net.RPCAddr = "0:26535" - cfgSlave.Storage.Path = "/tmp/rx_slave2" + cfgSlave.Storage.Path = "/tmp/reindex_slave2" os.RemoveAll(cfgSlave.Storage.Path) rxSlave := reindexer.NewReindex("builtinserver://xxx", reindexer.WithServerConfig(time.Second*100, cfgSlave)) { diff --git a/test/update_test.go b/test/update_test.go index 395d5ca69..32949b81f 100644 --- a/test/update_test.go +++ b/test/update_test.go @@ -13,13 +13,20 @@ import ( ) const ( - fieldsUpdateNs = "test_items_fields_update" - truncateNs = "test_truncate" - removeItemsNs = "test_remove_items" + fieldsUpdateNs = "test_items_fields_update" + truncateNs = "test_truncate" + removeItemsNs = "test_remove_items" + sparseArrItemNs = "sparse_array_updates" ) +type ItemWithSparseArray struct { + ID int64 `json:"id" reindex:"id,hash,pk"` + Array []int64 `json:"array_idx" reindex:"array_idx,hash,sparse"` +} + func init() { tnamespaces["test_items_insert_update"] = TestItemSimple{} + tnamespaces[sparseArrItemNs] = ItemWithSparseArray{} } var checkInsertUpdateExistsData = []*TestItemSimple{ @@ -1017,3 +1024,28 @@ func TestTruncateNamespace(t *testing.T) { require.NoError(t, DB.OpenNamespace(truncateNs, nsOpts, TestItemComplexObject{})) checkItemsCount(t, truncateNs, 0) } + +func TestUpdateSparseArrayIndex(t *testing.T) { + emptyItem := &ItemWithSparseArray{ + ID: int64(2), + } + + item := &ItemWithSparseArray{ + ID: int64(2), + Array: []int64{1, 2, 3}, + } + + require.NoError(t, DB.Upsert(sparseArrItemNs, emptyItem)) + + require.NoError(t, DB.Upsert(sparseArrItemNs, emptyItem)) + results := DB.ExecSQL("SELECT * FROM " + sparseArrItemNs + " WHERE id = 2") + checkResultItem(t, results, emptyItem) + + require.NoError(t, DB.Upsert(sparseArrItemNs, item)) + results = DB.ExecSQL("SELECT * FROM " + sparseArrItemNs + " WHERE id = 2") + checkResultItem(t, results, item) + + require.NoError(t, DB.Upsert(sparseArrItemNs, emptyItem)) + results = DB.ExecSQL("SELECT * FROM " + sparseArrItemNs + " WHERE id = 2") + checkResultItem(t, results, emptyItem) +} diff --git a/test/uuid_test.go b/test/uuid_test.go index 699cbab73..4316030e3 100644 --- a/test/uuid_test.go +++ b/test/uuid_test.go @@ -415,17 +415,17 @@ func TestUuidClientBuiltinserver(t *testing.T) { ns := "test_uuid_builtinserver_connect" t.Run("test add uuid index on non-indexed string", func(t *testing.T) { - path1 := "/tmp/rx_uuid11" + path1 := "/tmp/reindex_uuid11" rx1 := configureAndStartServer("0:29188", "0:26634", path1) defer rx1.Close() { - f, err := os.OpenFile("/tmp/rx_uuid11"+"/uudb/replication.conf", os.O_RDWR|os.O_CREATE, 0644) + f, err := os.OpenFile("/tmp/reindex_uuid11"+"/uudb/replication.conf", os.O_RDWR|os.O_CREATE, 0644) assert.NoError(t, err) _, err = f.Write([]byte(masterConfig)) assert.NoError(t, err) } - path2 := "/tmp/rx_uuid12" + path2 := "/tmp/reindex_uuid12" rx2 := configureAndStartServer("0:29189", "0:26635", path2) defer rx2.Close() { @@ -486,7 +486,7 @@ func TestUuidClientBuiltinserver(t *testing.T) { }) t.Run("test update index from string to uuid", func(t *testing.T) { - path1 := "/tmp/rx_uuid11" + path1 := "/tmp/reindex_uuid11" rx1 := configureAndStartServer("0:29188", "0:26634", path1) defer rx1.Close() { @@ -496,7 +496,7 @@ func TestUuidClientBuiltinserver(t *testing.T) { assert.NoError(t, err) } - path2 := "/tmp/rx_uuid12" + path2 := "/tmp/reindex_uuid12" rx2 := configureAndStartServer("0:29189", "0:26635", path2) defer rx2.Close() {