From 14aaa52ac9748c87c8c6eab0e9678972c6773651 Mon Sep 17 00:00:00 2001 From: reindexer-bot <@> Date: Fri, 15 Sep 2023 23:12:24 +0000 Subject: [PATCH] [fea] Add test for the CustomeLocale # Conflicts: # cpp_src/tools/customlocal.cc --- bindings/consts.go | 2 +- bindings/cproto/cproto.go | 30 +- bindings/cproto/cproto_test.go | 2 +- changelog.md | 50 +++ cjson/creflect.go | 50 ++- cjson/encoder.go | 418 ++++++++++++++--- cpp_src/CMakeLists.txt | 2 +- cpp_src/client/itemimpl.cc | 2 +- cpp_src/cmd/reindexer_server/main.cc | 7 +- .../test/test_storage_compatibility.sh | 194 ++++++++ cpp_src/cmd/reindexer_tool/reindexer_tool.cc | 19 +- cpp_src/core/cbinding/reindexer_c.cc | 4 +- cpp_src/core/cjson/baseencoder.cc | 13 +- cpp_src/core/cjson/baseencoder.h | 2 +- cpp_src/core/cjson/cjsondecoder.cc | 15 +- cpp_src/core/cjson/cjsondecoder.h | 11 +- cpp_src/core/cjson/cjsonmodifier.cc | 2 - cpp_src/core/cjson/cjsonmodifier.h | 2 +- cpp_src/core/cjson/cjsontools.cc | 11 +- cpp_src/core/cjson/cjsontools.h | 13 + cpp_src/core/cjson/fieldextractor.h | 13 +- cpp_src/core/cjson/jsondecoder.cc | 25 +- cpp_src/core/cjson/jsondecoder.h | 5 +- cpp_src/core/cjson/msgpackdecoder.cc | 43 +- cpp_src/core/cjson/msgpackdecoder.h | 5 +- cpp_src/core/cjson/protobufdecoder.cc | 23 +- cpp_src/core/cjson/protobufdecoder.h | 6 +- cpp_src/core/cjson/tagsmatcher.h | 12 +- cpp_src/core/cjson/tagsmatcherimpl.h | 6 +- cpp_src/core/comparatorimpl.h | 35 +- cpp_src/core/expressiontree.md | 31 +- cpp_src/core/ft/ft_fast/dataprocessor.cc | 2 +- cpp_src/core/ft/ft_fast/selecter.cc | 35 +- .../core/ft/ft_fuzzy/prefilter/prefilter.h | 1 + cpp_src/core/index/index.cc | 4 +- cpp_src/core/index/indexordered.cc | 5 +- cpp_src/core/index/indexstore.cc | 41 +- cpp_src/core/index/indexstore.h | 1 + cpp_src/core/index/indextext/fastindextext.cc | 91 ++-- cpp_src/core/index/indextext/fastindextext.h | 10 +- .../core/index/indextext/fuzzyindextext.cc | 12 +- cpp_src/core/index/indextext/fuzzyindextext.h | 4 +- cpp_src/core/index/indextext/indextext.cc | 14 +- cpp_src/core/index/indexunordered.cc | 5 +- cpp_src/core/index/uuid_index.h | 1 - cpp_src/core/indexdef.cc | 13 +- cpp_src/core/item.cc | 32 +- cpp_src/core/item.h | 10 +- cpp_src/core/itemimpl.cc | 6 +- cpp_src/core/itemmodifier.cc | 39 +- cpp_src/core/itemmodifier.h | 8 +- cpp_src/core/keyvalue/geometry.h | 4 +- cpp_src/core/keyvalue/uuid.cc | 276 +++++++++--- cpp_src/core/keyvalue/variant.cc | 12 +- cpp_src/core/keyvalue/variant.h | 17 +- cpp_src/core/lrucache.cc | 12 +- cpp_src/core/lrucache.h | 2 +- cpp_src/core/namespace/bgnamespacedeleter.h | 32 ++ cpp_src/core/namespace/namespace.cc | 9 +- cpp_src/core/namespace/namespace.h | 100 +++-- cpp_src/core/namespace/namespaceimpl.cc | 306 ++++++------- cpp_src/core/namespace/namespaceimpl.h | 60 ++- cpp_src/core/nsselecter/explaincalc.cc | 222 ++++++--- cpp_src/core/nsselecter/explaincalc.h | 56 ++- cpp_src/core/nsselecter/joinedselector.cc | 4 +- cpp_src/core/nsselecter/joinedselector.h | 14 +- cpp_src/core/nsselecter/nsselecter.cc | 138 ++++-- cpp_src/core/nsselecter/nsselecter.h | 6 +- cpp_src/core/nsselecter/querypreprocessor.cc | 254 ++++++++++- cpp_src/core/nsselecter/querypreprocessor.h | 18 +- cpp_src/core/nsselecter/selectiterator.cc | 5 +- cpp_src/core/nsselecter/selectiterator.h | 4 +- .../nsselecter/selectiteratorcontainer.cc | 33 +- cpp_src/core/nsselecter/substitutionhelpers.h | 2 +- cpp_src/core/payload/fieldsset.h | 2 + cpp_src/core/payload/payloadiface.cc | 2 +- cpp_src/core/query/dsl/dslencoder.cc | 1 + cpp_src/core/query/dsl/dslencoder.h | 3 +- cpp_src/core/query/dsl/dslparser.cc | 11 +- cpp_src/core/query/expressionevaluator.cc | 307 ++++++++----- cpp_src/core/query/expressionevaluator.h | 24 +- cpp_src/core/query/query.cc | 8 +- cpp_src/core/query/query.h | 6 +- cpp_src/core/query/queryentry.cc | 55 +++ cpp_src/core/query/queryentry.h | 48 +- cpp_src/core/query/sql/sqlencoder.cc | 20 +- cpp_src/core/query/sql/sqlencoder.h | 5 - cpp_src/core/query/sql/sqlparser.cc | 42 +- cpp_src/core/queryresults/aggregationresult.h | 14 + cpp_src/core/queryresults/queryresults.cc | 59 ++- cpp_src/core/queryresults/queryresults.h | 24 +- cpp_src/core/reindexerimpl.cc | 157 ++++--- cpp_src/core/reindexerimpl.h | 6 +- cpp_src/core/selectfunc/functionexecutor.cc | 7 +- cpp_src/core/selectfunc/functionexecutor.h | 2 +- cpp_src/core/selectfunc/selectfuncparser.cc | 29 +- cpp_src/core/selectfunc/selectfuncparser.h | 11 +- cpp_src/core/type_consts.h | 1 + cpp_src/core/type_consts_helpers.cc | 32 +- cpp_src/core/type_consts_helpers.h | 32 +- cpp_src/debug/allocdebug.h | 2 +- cpp_src/debug/backtrace.cc | 4 +- cpp_src/debug/backtrace.h | 2 +- cpp_src/debug/terminate_handler.cpp | 2 +- cpp_src/estl/defines.h | 4 +- cpp_src/estl/intrusive_ptr.h | 39 +- cpp_src/estl/tokenizer.cc | 53 +-- cpp_src/estl/tokenizer.h | 68 +-- cpp_src/gtests/tests/API/base_tests.cc | 94 +++- .../gtests/tests/fixtures/join_selects_api.h | 15 + cpp_src/gtests/tests/fixtures/ns_api.h | 31 ++ cpp_src/gtests/tests/fixtures/queries_api.cc | 190 ++++++++ cpp_src/gtests/tests/fixtures/queries_api.h | 61 ++- cpp_src/gtests/tests/fixtures/ttl_index_api.h | 2 +- .../gtests/tests/unit/composite_indexes_api.h | 10 +- .../tests/unit/composite_indexes_test.cc | 48 ++ cpp_src/gtests/tests/unit/ft/ft_generic.cc | 2 +- cpp_src/gtests/tests/unit/join_test.cc | 31 ++ cpp_src/gtests/tests/unit/namespace_test.cc | 319 +++++++++---- cpp_src/gtests/tests/unit/queries_test.cc | 1 + .../gtests/tests/unit/string_function_test.cc | 33 +- .../gtests/tests/unit/value_by_json_path.cc | 31 -- cpp_src/net/connection.cc | 14 +- cpp_src/net/cproto/dispatcher.h | 2 +- cpp_src/net/cproto/serverconnection.cc | 112 +++-- cpp_src/net/cproto/serverconnection.h | 4 +- cpp_src/net/ev/ev.h | 6 +- cpp_src/net/socket.cc | 20 + cpp_src/net/socket.h | 5 +- cpp_src/replicator/replicator.cc | 20 +- cpp_src/server/CMakeLists.txt | 2 +- cpp_src/server/contrib/server.md | 33 +- cpp_src/server/contrib/server.yml | 69 ++- cpp_src/server/pprof/pprof.cc | 2 +- cpp_src/server/rpcserver.cc | 9 +- cpp_src/server/rpcupdatespusher.cc | 3 +- cpp_src/server/server.cc | 8 +- cpp_src/server/server.h | 8 +- cpp_src/server/serverimpl.cc | 22 +- cpp_src/server/serverimpl.h | 10 +- cpp_src/tools/customhash.cc | 2 +- cpp_src/tools/customlocal.cc | 353 ++++++++++----- cpp_src/tools/customlocal.h | 7 +- cpp_src/tools/logger.cc | 66 ++- cpp_src/tools/logger.h | 4 +- cpp_src/tools/logginglongqueries.cc | 54 ++- cpp_src/tools/logginglongqueries.h | 96 +++- cpp_src/tools/md5crypt.cc | 6 +- cpp_src/tools/stringstools.cc | 33 +- cpp_src/tools/stringstools.h | 26 +- cpp_src/tools/timetools.cc | 16 +- cpp_src/tools/timetools.h | 1 + cpp_src/vendor/sort/pdqsort.hpp | 1 + cpp_src/vendor/sparse-map/sparse_hash.h | 3 +- iterator.go | 46 ++ query.go | 4 +- readme.md | 23 +- test/composite_indexes_test.go | 425 ++++++++++-------- test/eq_and_set_test.go | 39 +- test/huge_items_test.go | 73 +-- test/join_test.go | 114 +++++ test/queries_test.go | 30 +- test/query_test.go | 9 + test/tx_test.go | 30 ++ test/uuid_test.go | 92 +++- tx.go | 53 ++- 166 files changed, 4908 insertions(+), 1970 deletions(-) create mode 100644 cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh create mode 100644 cpp_src/core/namespace/bgnamespacedeleter.h diff --git a/bindings/consts.go b/bindings/consts.go index d676bf95f..df37a81f7 100644 --- a/bindings/consts.go +++ b/bindings/consts.go @@ -2,7 +2,7 @@ package bindings const CInt32Max = int(^uint32(0) >> 1) -const ReindexerVersion = "v3.18.0" +const ReindexerVersion = "v3.19.0" // public go consts from type_consts.h and reindexer_ctypes.h const ( diff --git a/bindings/cproto/cproto.go b/bindings/cproto/cproto.go index 5aa77d9bf..bc6ab855b 100644 --- a/bindings/cproto/cproto.go +++ b/bindings/cproto/cproto.go @@ -18,8 +18,9 @@ import ( const ( defConnPoolSize = 8 - defConnPoolLBAlgorithm = bindings.LBRoundRobin - pingerTimeoutSec = 60 + defConnPoolLBAlgorithm = bindings.LBPowerOfTwoChoices + pingerTimeoutSec = uint32(60) + pingResponseTimeoutSec = uint32(20) defAppName = "Go-connector" opRd = 0 @@ -586,7 +587,7 @@ func (binding *NetCProto) rpcCallNoResults(ctx context.Context, op int, cmd int, func (binding *NetCProto) pinger() { timeout := time.Second ticker := time.NewTicker(timeout) - var ticksCount uint16 + var ticksCount uint32 for now := range ticker.C { ticksCount++ select { @@ -597,15 +598,32 @@ func (binding *NetCProto) pinger() { if ticksCount == pingerTimeoutSec { ticksCount = 0 conns := binding.getAllConns() + var wg sync.WaitGroup + cmpl := func(buf bindings.RawBuffer, err error) { + wg.Done() + if buf != nil { + buf.Free() + } + } for _, conn := range conns { if conn.hasError() { continue } - if conn.lastReadTime().Add(timeout).Before(now) { - buf, _ := conn.rpcCall(context.TODO(), cmdPing, uint32(binding.timeouts.RequestTimeout*time.Second)) - buf.Free() + if !conn.lastReadTime().Add(timeout).Before(now) { + continue } + if cap(conn.seqs)-len(conn.seqs) > 0 { + continue + } + timeout := pingResponseTimeoutSec + if uint32(binding.timeouts.RequestTimeout) > timeout { + timeout = uint32(binding.timeouts.RequestTimeout) + } + + wg.Add(1) + conn.rpcCallAsync(context.TODO(), cmdPing, uint32(timeout), cmpl) } + wg.Wait() } } } diff --git a/bindings/cproto/cproto_test.go b/bindings/cproto/cproto_test.go index d06a027e1..dcdc531cd 100644 --- a/bindings/cproto/cproto_test.go +++ b/bindings/cproto/cproto_test.go @@ -74,7 +74,7 @@ func TestCprotoPool(t *testing.T) { u, err := url.Parse(dsn) require.NoError(t, err) c := new(NetCProto) - err = c.Init([]url.URL{*u}) + err = c.Init([]url.URL{*u}, reindexer.WithConnPoolLoadBalancing(bindings.LBRoundRobin)) require.NoError(t, err) conns := make(map[*connection]bool) diff --git a/changelog.md b/changelog.md index 725d77331..9c3b0da69 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,53 @@ +# Version 3.19.0 (16.09.2023) +## Core +- [fea] Added background namespaces deleteion. Previously in some cases atomic transactions could cause the namespace's deletion in the querie's execution thread, which lead to the latency spike +- [fea] Added locks and flush timings for UPDATE and DELETE queries into [slow log](readme.md#slow-actions-logging) +- [fea] Added support for the [COUNT/COUNT_CACHED](readme.md#aggregations) aggregations for the MERGE-queries +- [fea] Added basic support for the SUM/MAX/MIN aggregations for the MERGE-queries without LIMIT/OFFSET +- [fea] Added more information about JOINs condtitions injection into `explain` (check `on_conditions_injections` field in the explain results) +- [fea] Added `field_type` into the `explain`. This fields shows, which kind of field/comparator was used in the query: `indexed` or `non-indexed` +- [fea] Added support for store (`-`) UUID indexes +- [fea] Optimized string<->UUID convertions in the C++ library +- [fix] Fixed `total_us` time calculation in the `explain` (previously `preselect` time was in the explain results, but was not counted in the `total_us` field) +- [fix] Fixed JOINs conditions injection for the queries with multiple JOINs. Previously this optimization could skip some of the JOIN queries +- [fix] Fixed multiple UPDATE arrays concatenation issues: empty arrays, string array, array with scalar, etc. +- [fix] Composite indexes over the other composite indexes were explicitly disabled +- [fix] Fixed check for nested JOINs and MERGEs (nested JOIN/MERGE is not allowed) +- [fix] Multiple JSON-paths were explicitly disabled for the scalar indexes (multiple JSON-paths are allowed for composite and array indexes only) +- [fix] Fixed crash on the CJSON deserialization for array indexes with multiple JSON-paths + +## Replication +- [fix] Fixed handling of the updates buffer overflow (previously this logic may lead to multiple extra resyncs) + +## Fulltext +- [fea] Added experimental support for Armenian, Hebrew, Arabic and Devanagari alphabets (prefixes/postfixes/suffixes search, typos, stop-words and synonyms are supported) + +## Reindexer server +- [fix] Fixed SSE4.2 support check on startup (server has to print error message if SSE support is required for the current build) +- [fix] Fixed DELETE-queries compatibility between `reindexer_server v3` and `reindexer_tool v4` in cproto mode +- [fix] Fixed RPC-timeout handling (extra check for unread data wasd added). Previously connect could be dropped in case of the some very long queries + +## Go connector +- [fea] Optimized string<->UUID convertions in the Go bindings +- [fea] Default balancing algorithm for `cproto` was changed from RoundRobin to PowerOfTwoChoices. RoundRobin may still be enabled via `WithConnPoolLoadBalancing` option of the binding +- [fix] Fixed transactions `Commit`/`Rollback` stucking in case of the incomatible items' sctructs in client and DB +- [fix] Fixed `builtinserver` logger initialization in cases when single app run multiple instances of the `builtinserver` +- [fix] Fixed RPC-connections background pinging (Ping requests became asynchronous) + +## Build +- [fix] Fixed build with GCC 13.2 + +## Face +- [fea] Changed the numeric values position in the Grid +- [fea] Added the ability to use spaces for JSON paths +- [fix] Fixed the Statistics chart for undefined replication.wal_size +- [fix] Fixed the column set for the namespace items during the namespace switching +- [fix] Fixed the JSON paths view for values included spaces +- [fix] Changed the value format for width on the integer for sqlquery +- [fix] Fixed the bug related to the query history on the Namespace Items list +- [fix] Fixed the column titles in the table settings menu on the Performance page +- [fix] Added the validation of the negative values for the index settings + # Version 3.18.0 (29.07.2023) ## Core - [fea] Increased max indexes count for each namespace up to 255 user-defined indexes (previously it was 63) diff --git a/cjson/creflect.go b/cjson/creflect.go index 5ed7611b9..78c2e9161 100644 --- a/cjson/creflect.go +++ b/cjson/creflect.go @@ -3,7 +3,6 @@ package cjson import ( "fmt" "reflect" - "strings" "unsafe" "github.com/restream/reindexer/v3/bindings" @@ -107,17 +106,44 @@ func (pl *payloadIface) ptr(field, idx, typ int) unsafe.Pointer { const hexChars = "0123456789abcdef" func createUuid(v [2]uint64) string { - var b strings.Builder - b.Grow(36) - for i, j := 0, 0; i < 36; i++ { - switch i { - case 8, 13, 18, 23: b.WriteByte('-') - default: - b.WriteByte(hexChars[(v[j / 16] >> ((15 - j % 16) * 4)) & 0xF]) - j++ - } - } - return b.String() + buf := make([]byte, 36) + buf[0] = hexChars[(v[0] >> 60) & 0xF]; + buf[1] = hexChars[(v[0] >> 56) & 0xF]; + buf[2] = hexChars[(v[0] >> 52) & 0xF]; + buf[3] = hexChars[(v[0] >> 48) & 0xF]; + buf[4] = hexChars[(v[0] >> 44) & 0xF]; + buf[5] = hexChars[(v[0] >> 40) & 0xF]; + buf[6] = hexChars[(v[0] >> 36) & 0xF]; + buf[7] = hexChars[(v[0] >> 32) & 0xF]; + buf[8] = '-'; + buf[9] = hexChars[(v[0] >> 28) & 0xF]; + buf[10] = hexChars[(v[0] >> 24) & 0xF]; + buf[11] = hexChars[(v[0] >> 20) & 0xF]; + buf[12] = hexChars[(v[0] >> 16) & 0xF]; + buf[13] = '-'; + buf[14] = hexChars[(v[0] >> 12) & 0xF]; + buf[15] = hexChars[(v[0] >> 8) & 0xF]; + buf[16] = hexChars[(v[0] >> 4) & 0xF]; + buf[17] = hexChars[v[0] & 0xF]; + buf[18] = '-'; + buf[19] = hexChars[(v[1] >> 60) & 0xF]; + buf[20] = hexChars[(v[1] >> 56) & 0xF]; + buf[21] = hexChars[(v[1] >> 52) & 0xF]; + buf[22] = hexChars[(v[1] >> 48) & 0xF]; + buf[23] = '-'; + buf[24] = hexChars[(v[1] >> 44) & 0xF]; + buf[25] = hexChars[(v[1] >> 40) & 0xF]; + buf[26] = hexChars[(v[1] >> 36) & 0xF]; + buf[27] = hexChars[(v[1] >> 32) & 0xF]; + buf[28] = hexChars[(v[1] >> 28) & 0xF]; + buf[29] = hexChars[(v[1] >> 24) & 0xF]; + buf[30] = hexChars[(v[1] >> 20) & 0xF]; + buf[31] = hexChars[(v[1] >> 16) & 0xF]; + buf[32] = hexChars[(v[1] >> 12) & 0xF]; + buf[33] = hexChars[(v[1] >> 8) & 0xF]; + buf[34] = hexChars[(v[1] >> 4) & 0xF]; + buf[35] = hexChars[v[1] & 0xF]; + return string(buf) } func (pl *payloadIface) getInt(field, idx int) int { diff --git a/cjson/encoder.go b/cjson/encoder.go index d0630282d..ab40554c9 100644 --- a/cjson/encoder.go +++ b/cjson/encoder.go @@ -223,67 +223,375 @@ func (enc *Encoder) encodeMap(v reflect.Value, rdser *Serializer, idx []int) err return nil } -func ParseUuid(s string) (res [2]uint64, err error) { - if len(s) == 0 { - return +var hexCharToUint = [256]uint64{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +} + +func generateError(ch byte, str string) (res [2]uint64, err error) { + if ch == '-' { + err = fmt.Errorf("Invalid UUID format: '%s'", str) + } else { + err = fmt.Errorf("UUID cannot contain char '%c': '%s'", ch, str) } - i := 0 - for _, ch := range s { - if i >= 32 && ch != '-' { - err = fmt.Errorf("UUID should consist of 32 hexadecimal digits: '%s'", s) - return + return +} + +func ParseUuid(str string) (res [2]uint64, err error) { + switch len(str) { + case 0: + return + case 32: + ch := str[0] + num := hexCharToUint[ch] + if num > 15 { + return generateError(ch, str) } - var v uint64 - switch ch { - case '0': - v = 0 - case '1': - v = 1 - case '2': - v = 2 - case '3': - v = 3 - case '4': - v = 4 - case '5': - v = 5 - case '6': - v = 6 - case '7': - v = 7 - case '8': - v = 8 - case '9': - v = 9 - case 'a', 'A': - v = 10 - case 'b', 'B': - v = 11 - case 'c', 'C': - v = 12 - case 'd', 'D': - v = 13 - case 'e', 'E': - v = 14 - case 'f', 'F': - v = 15 - case '-': - continue - default: - err = fmt.Errorf("UUID cannot contain char '%c': '%s'", ch, s) + res[0] = num << 60 + ch = str[1] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 56 + ch = str[2] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 52 + ch = str[3] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 48 + ch = str[4] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 44 + ch = str[5] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 40 + ch = str[6] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 36 + ch = str[7] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 32 + ch = str[8] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 28 + ch = str[9] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 24 + ch = str[10] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 20 + ch = str[11] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 16 + ch = str[12] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 12 + ch = str[13] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 8 + ch = str[14] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 4 + ch = str[15] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num + ch = str[16] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] = num << 60 + ch = str[17] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 56 + ch = str[18] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 52 + ch = str[19] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 48 + ch = str[20] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 44 + ch = str[21] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 40 + ch = str[22] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 36 + ch = str[23] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 32 + ch = str[24] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 28 + ch = str[25] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 24 + ch = str[26] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 20 + ch = str[27] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 16 + ch = str[28] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 12 + ch = str[29] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 8 + ch = str[30] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 4 + ch = str[31] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num + break + case 36: + if str[8] != '-' || str[13] != '-' || str[18] != '-' || str[23] != '-' { + err = fmt.Errorf("Invalid UUID format: '%s'", str) return } - res[i/16] = (res[i/16] << 4) | v - i++ - } - if i != 32 { - err = fmt.Errorf("UUID should consist of 32 hexadecimal digits: '%s'", s) - return - } - if (res[0] != 0 || res[1] != 0) && (res[1]>>63) == 0 { - err = fmt.Errorf("Variant 0 of UUID is unsupported: '%s'", s) + ch := str[0] + num := hexCharToUint[ch] + if num > 15 { + return generateError(ch, str) + } + res[0] = num << 60 + ch = str[1] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 56 + ch = str[2] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 52 + ch = str[3] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 48 + ch = str[4] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 44 + ch = str[5] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 40 + ch = str[6] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 36 + ch = str[7] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 32 + ch = str[9] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 28 + ch = str[10] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 24 + ch = str[11] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 20 + ch = str[12] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 16 + ch = str[14] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 12 + ch = str[15] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 8 + ch = str[16] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num << 4 + ch = str[17] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[0] |= num + ch = str[19] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] = num << 60 + ch = str[20] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 56 + ch = str[21] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 52 + ch = str[22] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 48 + ch = str[24] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 44 + ch = str[25] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 40 + ch = str[26] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 36 + ch = str[27] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 32 + ch = str[28] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 28 + ch = str[29] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 24 + ch = str[30] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 20 + ch = str[31] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 16 + ch = str[32] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 12 + ch = str[33] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 8 + ch = str[34] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num << 4 + ch = str[35] + if num = hexCharToUint[ch]; num > 15 { + return generateError(ch, str) + } + res[1] |= num + break + default: + err = fmt.Errorf("UUID should consist of 32 hexadecimal digits: '%s'", str) return } + if (res[0] != 0 || res[1] != 0) && (res[1] >> 63) == 0 { + err = fmt.Errorf("Variant 0 of UUID is unsupported: '%s'", str) + } return } diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index 754f49e93..df73c53cd 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -36,7 +36,7 @@ else() option (LINK_RESOURCES "Link web resources as binary data" ON) endif() -set (REINDEXER_VERSION_DEFAULT "3.18.0") +set (REINDEXER_VERSION_DEFAULT "3.19.0") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") diff --git a/cpp_src/client/itemimpl.cc b/cpp_src/client/itemimpl.cc index 98f32d0a1..3d33bf1db 100644 --- a/cpp_src/client/itemimpl.cc +++ b/cpp_src/client/itemimpl.cc @@ -96,7 +96,7 @@ Error ItemImpl::FromJSON(std::string_view slice, char **endp, bool /*pkOnly*/) { Error ItemImpl::FromMsgPack(std::string_view buf, size_t &offset) { Payload pl = GetPayload(); - MsgPackDecoder decoder(&tagsMatcher_); + MsgPackDecoder decoder(tagsMatcher_); ser_.Reset(); Error err = decoder.Decode(buf, pl, ser_, offset); diff --git a/cpp_src/cmd/reindexer_server/main.cc b/cpp_src/cmd/reindexer_server/main.cc index 4f3c4dc01..11cf822d8 100644 --- a/cpp_src/cmd/reindexer_server/main.cc +++ b/cpp_src/cmd/reindexer_server/main.cc @@ -7,7 +7,12 @@ int main(int argc, char* argv[]) { reindexer::debug::backtrace_init(); - reindexer::CheckRequiredSSESupport(); + try { + reindexer::CheckRequiredSSESupport(); + } catch (Error& err) { + std::cerr << err.what(); + return EXIT_FAILURE; + } reindexer_server::Server svc(reindexer_server::ServerMode::Standalone); auto err = svc.InitFromCLI(argc, argv); diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100644 index 000000000..2c014c6a5 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-server-.*.rpm' | tail -n1 | cut -c 2-) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-4-server-.*.rpm' | tail -n1 | cut -c 2-) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip; +unzip -o big.zip # unzips into mydb_big.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"replication":{"data_hash":24651210926,"data_count":3}}, +{"replication":{"data_hash":6252344969,"data_count":1}}, +{"replication":{"data_hash":37734732881,"data_count":28}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":1024095024522,"data_count":1145}}, +{"replication":{"data_hash":8373644068,"data_count":1315}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":7404222244,"data_count":97}}, +{"replication":{"data_hash":94132837196,"data_count":4}}, +{"replication":{"data_hash":1896088071,"data_count":2}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":-672103903,"data_count":33538}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":6833710705,"data_count":1}}, +{"replication":{"data_hash":5858155773472,"data_count":4500}}, +{"replication":{"data_hash":-473221280268823592,"data_count":65448}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":8288213744,"data_count":3}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":354171024786967,"data_count":3941}}, +{"replication":{"data_hash":-6520334670,"data_count":35886}}, +{"replication":{"data_hash":112772074632,"data_count":281}}, +{"replication":{"data_hash":-12679568198538,"data_count":1623116}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/cmd/reindexer_tool/reindexer_tool.cc b/cpp_src/cmd/reindexer_tool/reindexer_tool.cc index cd5229406..cb5be7eb0 100644 --- a/cpp_src/cmd/reindexer_tool/reindexer_tool.cc +++ b/cpp_src/cmd/reindexer_tool/reindexer_tool.cc @@ -30,11 +30,13 @@ static void InstallLogLevel(const std::vector& args) { llevel = 3; } - reindexer::logInstallWriter([](int level, char* buf) { - if (level <= llevel) { - std::cout << buf << std::endl; - } - }); + reindexer::logInstallWriter( + [](int level, char* buf) { + if (level <= llevel) { + std::cout << buf << std::endl; + } + }, + reindexer::LoggerPolicy::WithoutLocks); } } // namespace reindexer_tool @@ -43,7 +45,12 @@ int main(int argc, char* argv[]) { using namespace reindexer_tool; reindexer::debug::backtrace_init(); - reindexer::CheckRequiredSSESupport(); + try { + reindexer::CheckRequiredSSESupport(); + } catch (Error& err) { + std::cerr << err.what(); + return EXIT_FAILURE; + } args::ArgumentParser parser("Reindexer client tool"); args::HelpFlag help(parser, "help", "show this message", {'h', "help"}); diff --git a/cpp_src/core/cbinding/reindexer_c.cc b/cpp_src/core/cbinding/reindexer_c.cc index 5e601f05c..134937df5 100644 --- a/cpp_src/core/cbinding/reindexer_c.cc +++ b/cpp_src/core/cbinding/reindexer_c.cc @@ -694,9 +694,9 @@ reindexer_error reindexer_commit(uintptr_t rx, reindexer_string nsName) { return error2c(!db ? err_not_init : db->Commit(str2cv(nsName))); } -void reindexer_enable_logger(void (*logWriter)(int, char*)) { logInstallWriter(logWriter, false); } +void reindexer_enable_logger(void (*logWriter)(int, char*)) { logInstallWriter(logWriter, LoggerPolicy::WithLocks); } -void reindexer_disable_logger() { logInstallWriter(nullptr, false); } +void reindexer_disable_logger() { logInstallWriter(nullptr, LoggerPolicy::WithLocks); } reindexer_error reindexer_free_buffer(reindexer_resbuffer in) { constexpr static put_results_to_pool putResultsToPool; diff --git a/cpp_src/core/cjson/baseencoder.cc b/cpp_src/core/cjson/baseencoder.cc index 2314aa3eb..9dd542c94 100644 --- a/cpp_src/core/cjson/baseencoder.cc +++ b/cpp_src/core/cjson/baseencoder.cc @@ -146,14 +146,10 @@ bool BaseEncoder::encode(ConstPayload* pl, Serializer& rdser, Builder& // get field from indexed field if (tagField >= 0) { if (!pl) throw Error(errParams, "Trying to encode index field %d without payload", tagField); - if (objectScalarIndexes_.test(tagField) && (tagType != TAG_ARRAY)) { - std::string fieldName; - if (tagName && tagsMatcher_) { - fieldName = tagsMatcher_->tag2name(tagName); - } - throw Error(errParams, "Non-array field '%s' [%d] from '%s' can only be encoded once.", fieldName, tagField, pl->Type().Name()); + const auto& f = pl->Type().Field(tagField); + if (!f.IsArray() && objectScalarIndexes_.test(tagField)) { + throw Error(errParams, "Non-array field '%s' [%d] from '%s' can only be encoded once.", f.Name(), tagField, pl->Type().Name()); } - objectScalarIndexes_.set(tagField); assertrx(tagField < pl->NumFields()); int* cnt = &fieldsoutcnt_[tagField]; switch (tagType) { @@ -176,6 +172,7 @@ bool BaseEncoder::encode(ConstPayload* pl, Serializer& rdser, Builder& break; } case TAG_NULL: + objectScalarIndexes_.set(tagField); if (visible) builder.Null(tagName); break; case TAG_VARINT: @@ -185,6 +182,7 @@ bool BaseEncoder::encode(ConstPayload* pl, Serializer& rdser, Builder& case TAG_END: case TAG_OBJECT: case TAG_UUID: + objectScalarIndexes_.set(tagField); if (visible) builder.Put(tagName, pl->Get(tagField, (*cnt)), *cnt); ++(*cnt); break; @@ -217,7 +215,6 @@ bool BaseEncoder::encode(ConstPayload* pl, Serializer& rdser, Builder& break; } case TAG_OBJECT: { - objectScalarIndexes_.reset(); if (visible) { auto objNode = builder.Object(tagName); while (encode(pl, rdser, objNode, true)) diff --git a/cpp_src/core/cjson/baseencoder.h b/cpp_src/core/cjson/baseencoder.h index e08c66feb..0e2536316 100644 --- a/cpp_src/core/cjson/baseencoder.h +++ b/cpp_src/core/cjson/baseencoder.h @@ -69,7 +69,7 @@ class BaseEncoder { TagsPath curTagsPath_; IndexedTagsPathInternalT indexedTagsPath_; TagsLengths tagsLengths_; - std::bitset objectScalarIndexes_; + ScalarIndexesSetT objectScalarIndexes_; }; using JsonEncoder = BaseEncoder; diff --git a/cpp_src/core/cjson/cjsondecoder.cc b/cpp_src/core/cjson/cjsondecoder.cc index 9db833d12..b74b2bcba 100644 --- a/cpp_src/core/cjson/cjsondecoder.cc +++ b/cpp_src/core/cjson/cjsondecoder.cc @@ -7,10 +7,6 @@ namespace reindexer { -CJsonDecoder::CJsonDecoder(TagsMatcher &tagsMatcher) : tagsMatcher_(tagsMatcher), filter_(nullptr) {} -CJsonDecoder::CJsonDecoder(TagsMatcher &tagsMatcher, const FieldsSet *filter, Recoder *recoder) - : tagsMatcher_(tagsMatcher), filter_(filter), recoder_{recoder} {} - bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrser, bool match) { const ctag tag = rdser.GetCTag(); TagType tagType = tag.Type(); @@ -24,7 +20,7 @@ bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrs (void)tagsMatcher_.tag2name(tagName); tagsPath_.emplace_back(tagName); } - if (tag.Field() >= 0) { + if rx_unlikely (tag.Field() >= 0) { throw Error(errLogic, "Reference tag was found in transport CJSON for field %d[%s] in ns [%s]", tag.Field(), tagsMatcher_.tag2name(tagName), pl.Type().Name()); } @@ -56,6 +52,7 @@ bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrs if (field >= 0) { if (match) { if (tagType == TAG_NULL) { + objectScalarIndexes_.set(field); wrser.PutCTag(ctag{TAG_NULL, tagName}); } else if (recoder) { recoder->Recode(rdser, pl, tagName, wrser); @@ -63,7 +60,7 @@ bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrs const auto &fieldRef{pl.Type().Field(field)}; const KeyValueType fieldType{fieldRef.Type()}; if (tagType == TAG_ARRAY) { - if (!fieldRef.IsArray()) { + if rx_unlikely (!fieldRef.IsArray()) { throw Error(errLogic, "Error parsing cjson field '%s' - got array, expected scalar %s", fieldRef.Name(), fieldType.Name()); } @@ -77,10 +74,9 @@ bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrs } wrser.PutCTag(ctag{TAG_ARRAY, tagName, field}); wrser.PutVarUint(count); - } else if (isInArray() && !fieldRef.IsArray()) { - throw Error(errLogic, "Error parsing cjson field '%s' - got value in the nested array, but expected scalar %s", - fieldRef.Name(), fieldType.Name()); } else { + validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, fieldRef, field, isInArray(), "cjson"); + objectScalarIndexes_.set(field); pl.Set(field, cjsonValueToVariant(tagType, rdser, fieldType), true); fieldType.EvaluateOneOf( [&](OneOf) { @@ -93,6 +89,7 @@ bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrs } } } else { + // objectScalarIndexes_.set(field); - do not change objectScalarIndexes_ value for the filtered out fields skipCjsonTag(tag, rdser); } } else { diff --git a/cpp_src/core/cjson/cjsondecoder.h b/cpp_src/core/cjson/cjsondecoder.h index 35ebfb0d9..a128fe487 100644 --- a/cpp_src/core/cjson/cjsondecoder.h +++ b/cpp_src/core/cjson/cjsondecoder.h @@ -20,10 +20,14 @@ class Recoder { class CJsonDecoder { public: - CJsonDecoder(TagsMatcher &tagsMatcher); - CJsonDecoder(TagsMatcher &tagsMatcher, const FieldsSet *filter, Recoder *); + CJsonDecoder(TagsMatcher &tagsMatcher) noexcept : tagsMatcher_(tagsMatcher), filter_(nullptr) {} + CJsonDecoder(TagsMatcher &tagsMatcher, const FieldsSet *filter, Recoder *recoder) noexcept + : tagsMatcher_(tagsMatcher), filter_(filter), recoder_(recoder) {} - void Decode(Payload &pl, Serializer &rdSer, WrSerializer &wrSer) { decodeCJson(pl, rdSer, wrSer, true); } + void Decode(Payload &pl, Serializer &rdSer, WrSerializer &wrSer) { + objectScalarIndexes_.reset(); + decodeCJson(pl, rdSer, wrSer, true); + } private: bool decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrser, bool match); @@ -34,6 +38,7 @@ class CJsonDecoder { TagsPath tagsPath_; Recoder *recoder_{nullptr}; int32_t arrayLevel_ = 0; + ScalarIndexesSetT objectScalarIndexes_; }; } // namespace reindexer diff --git a/cpp_src/core/cjson/cjsonmodifier.cc b/cpp_src/core/cjson/cjsonmodifier.cc index d549ce1a8..8409dbeeb 100644 --- a/cpp_src/core/cjson/cjsonmodifier.cc +++ b/cpp_src/core/cjson/cjsonmodifier.cc @@ -45,8 +45,6 @@ class CJsonModifier::Context { bool isForAllItems_ = false; }; -CJsonModifier::CJsonModifier(TagsMatcher &tagsMatcher, PayloadType pt) : pt_(std::move(pt)), tagsMatcher_(tagsMatcher) {} - void CJsonModifier::SetFieldValue(std::string_view tuple, IndexedTagsPath fieldPath, const VariantArray &val, WrSerializer &ser, const Payload &pl) { if (fieldPath.empty()) { diff --git a/cpp_src/core/cjson/cjsonmodifier.h b/cpp_src/core/cjson/cjsonmodifier.h index 68ed342c3..642e2908b 100644 --- a/cpp_src/core/cjson/cjsonmodifier.h +++ b/cpp_src/core/cjson/cjsonmodifier.h @@ -6,7 +6,7 @@ namespace reindexer { class CJsonModifier { public: - CJsonModifier(TagsMatcher &tagsMatcher, PayloadType pt); + CJsonModifier(TagsMatcher &tagsMatcher, PayloadType pt) noexcept : pt_(std::move(pt)), tagsMatcher_(tagsMatcher) {} void SetFieldValue(std::string_view tuple, IndexedTagsPath path, const VariantArray &v, WrSerializer &ser, const Payload &pl); void SetObject(std::string_view tuple, IndexedTagsPath path, const VariantArray &v, WrSerializer &ser, const Payload &pl); void RemoveField(std::string_view tuple, IndexedTagsPath fieldPath, WrSerializer &wrser); diff --git a/cpp_src/core/cjson/cjsontools.cc b/cpp_src/core/cjson/cjsontools.cc index 5669b0c87..0294772ff 100644 --- a/cpp_src/core/cjson/cjsontools.cc +++ b/cpp_src/core/cjson/cjsontools.cc @@ -140,9 +140,11 @@ Variant cjsonValueToVariant(TagType tagType, Serializer &rdser, KeyValueType dst template void buildPayloadTuple(const PayloadIface &pl, const TagsMatcher *tagsMatcher, WrSerializer &wrser) { CJsonBuilder builder(wrser, ObjType::TypeObject); - for (int field = 1; field < pl.NumFields(); ++field) { + for (int field = 1, numFields = pl.NumFields(); field < numFields; ++field) { const PayloadFieldType &fieldType = pl.Type().Field(field); - if (fieldType.JsonPaths().size() < 1 || fieldType.JsonPaths()[0].empty()) continue; + if (fieldType.JsonPaths().size() < 1 || fieldType.JsonPaths()[0].empty()) { + continue; + } int tagName = tagsMatcher->name2tag(fieldType.JsonPaths()[0]); assertf(tagName != 0, "ns=%s, field=%s", pl.Type().Name(), fieldType.JsonPaths()[0]); @@ -155,8 +157,7 @@ void buildPayloadTuple(const PayloadIface &pl, const TagsMatcher *tagsMatcher } } -template void buildPayloadTuple(const PayloadIface &pl, const TagsMatcher *tagsMatcher, - WrSerializer &wrser); -template void buildPayloadTuple(const PayloadIface &pl, const TagsMatcher *tagsMatcher, WrSerializer &wrser); +template void buildPayloadTuple(const PayloadIface &, const TagsMatcher *, WrSerializer &); +template void buildPayloadTuple(const PayloadIface &, const TagsMatcher *, WrSerializer &); } // namespace reindexer diff --git a/cpp_src/core/cjson/cjsontools.h b/cpp_src/core/cjson/cjsontools.h index 490633b36..69070c1e8 100644 --- a/cpp_src/core/cjson/cjsontools.h +++ b/cpp_src/core/cjson/cjsontools.h @@ -17,4 +17,17 @@ void putCJsonValue(TagType tagType, int tagName, const VariantArray &values, WrS void skipCjsonTag(ctag tag, Serializer &rdser, std::array *fieldsArrayOffsets = nullptr); [[nodiscard]] Variant cjsonValueToVariant(TagType tag, Serializer &rdser, KeyValueType dstType); +RX_ALWAYS_INLINE void validateNonArrayFieldRestrictions(const ScalarIndexesSetT &scalarIndexes, const Payload &pl, + const PayloadFieldType &f, int field, bool isInArray, std::string_view parserName) { + if (!f.IsArray()) { + if rx_unlikely (isInArray) { + throw Error(errLogic, "Error parsing %s field '%s' - got value nested into the array, but expected scalar %s", parserName, + f.Name(), f.Type().Name()); + } + if rx_unlikely (scalarIndexes.test(field)) { + throw Error(errLogic, "Non-array field '%s' [%d] from '%s' can only be encoded once.", f.Name(), field, pl.Type().Name()); + } + } +} + } // namespace reindexer diff --git a/cpp_src/core/cjson/fieldextractor.h b/cpp_src/core/cjson/fieldextractor.h index 1ef5eb5ec..6b11f179a 100644 --- a/cpp_src/core/cjson/fieldextractor.h +++ b/cpp_src/core/cjson/fieldextractor.h @@ -27,7 +27,10 @@ class FieldsExtractor { void SetTagsMatcher(const TagsMatcher *) noexcept {} FieldsExtractor Object(int) noexcept { return FieldsExtractor(values_, expectedType_, expectedPathDepth_ - 1, filter_, params_); } - FieldsExtractor Array(int) noexcept { return FieldsExtractor(values_, expectedType_, expectedPathDepth_ - 1, filter_, params_); } + FieldsExtractor Array(int) noexcept { + assertrx_throw(values_); + return FieldsExtractor(&values_->MarkArray(), expectedType_, expectedPathDepth_ - 1, filter_, params_); + } FieldsExtractor Object(std::string_view) noexcept { return FieldsExtractor(values_, expectedType_, expectedPathDepth_ - 1, filter_, params_); } @@ -67,6 +70,10 @@ class FieldsExtractor { put(0, Variant(d)); } } + if (expectedPathDepth_ <= 0) { + assertrx_throw(values_); + values_->MarkArray(); + } } void Array(int, Serializer &ser, TagType tagType, int count) { @@ -98,6 +105,10 @@ class FieldsExtractor { put(0, ser.GetRawVariant(KeyValueType(tagType))); } } + if (expectedPathDepth_ <= 0) { + assertrx_throw(values_); + values_->MarkArray(); + } } FieldsExtractor &Put(int t, Variant arg, int offset) { diff --git a/cpp_src/core/cjson/jsondecoder.cc b/cpp_src/core/cjson/jsondecoder.cc index bc9ab94d6..d10e82539 100644 --- a/cpp_src/core/cjson/jsondecoder.cc +++ b/cpp_src/core/cjson/jsondecoder.cc @@ -9,11 +9,9 @@ namespace reindexer { -JsonDecoder::JsonDecoder(TagsMatcher &tagsMatcher) : tagsMatcher_(tagsMatcher), filter_(nullptr) {} -JsonDecoder::JsonDecoder(TagsMatcher &tagsMatcher, const FieldsSet *filter) : tagsMatcher_(tagsMatcher), filter_(filter) {} - Error JsonDecoder::Decode(Payload &pl, WrSerializer &wrser, const gason::JsonValue &v) { try { + objectScalarIndexes_.reset(); tagsPath_.clear(); CJsonBuilder builder(wrser, ObjType::TypePlain, &tagsMatcher_); decodeJson(&pl, builder, v, 0, true); @@ -45,7 +43,7 @@ void JsonDecoder::decodeJsonObject(Payload &pl, CJsonBuilder &builder, const gas const auto &f = pl.Type().Field(field); switch (elem.value.getTag()) { case gason::JSON_ARRAY: { - if (!f.IsArray()) { + if rx_unlikely (!f.IsArray()) { throw Error(errLogic, "Error parsing json field '%s' - got array, expected scalar %s", f.Name(), f.Type().Name()); } int count = 0; @@ -60,10 +58,8 @@ void JsonDecoder::decodeJsonObject(Payload &pl, CJsonBuilder &builder, const gas builder.ArrayRef(tagName, field, count); } break; case gason::JSON_NULL: - if (isInArray() && !f.IsArray()) { - throw Error(errLogic, "Error parsing json field '%s' - got value in the nested array, but expected scalar %s", - f.Name(), f.Type().Name()); - } + validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, f, field, isInArray(), "json"); + objectScalarIndexes_.set(field); builder.Null(tagName); break; case gason::JSON_NUMBER: @@ -72,15 +68,15 @@ void JsonDecoder::decodeJsonObject(Payload &pl, CJsonBuilder &builder, const gas case gason::JSON_STRING: case gason::JSON_TRUE: case gason::JSON_FALSE: { - if (isInArray() && !f.IsArray()) { - throw Error(errLogic, "Error parsing json field '%s' - got value in the nested array, but expected scalar %s", - f.Name(), f.Type().Name()); - } + validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, f, field, isInArray(), "json"); + objectScalarIndexes_.set(field); Variant v = jsonValue2Variant(elem.value, f.Type(), f.Name()); - pl.Set(field, v, true); builder.Ref(tagName, v, field); + pl.Set(field, std::move(v), true); } break; } + } else { + // objectScalarIndexes_.set(field); - do not change objectScalarIndexes_ value for the filtered out fields } tagsPath_.pop_back(); } @@ -141,7 +137,7 @@ void JsonDecoder::decodeJson(Payload *pl, CJsonBuilder &builder, const gason::Js class TagsPathGuard { public: - TagsPathGuard(TagsPath &tagsPath, int tagName) : tagsPath_(tagsPath) { tagsPath_.emplace_back(tagName); } + TagsPathGuard(TagsPath &tagsPath, int tagName) noexcept : tagsPath_(tagsPath) { tagsPath_.emplace_back(tagName); } ~TagsPathGuard() { tagsPath_.pop_back(); } public: @@ -158,6 +154,7 @@ void JsonDecoder::decodeJsonObject(const gason::JsonValue &root, CJsonBuilder &b void JsonDecoder::Decode(std::string_view json, CJsonBuilder &builder, const TagsPath &fieldPath) { try { + objectScalarIndexes_.reset(); tagsPath_ = fieldPath; gason::JsonParser jsonParser; gason::JsonNode root = jsonParser.Parse(json); diff --git a/cpp_src/core/cjson/jsondecoder.h b/cpp_src/core/cjson/jsondecoder.h index d19e136f0..c44a1432c 100644 --- a/cpp_src/core/cjson/jsondecoder.h +++ b/cpp_src/core/cjson/jsondecoder.h @@ -8,8 +8,8 @@ namespace reindexer { class JsonDecoder { public: - explicit JsonDecoder(TagsMatcher &tagsMatcher); - JsonDecoder(TagsMatcher &tagsMatcher, const FieldsSet *filter); + explicit JsonDecoder(TagsMatcher &tagsMatcher) noexcept : tagsMatcher_(tagsMatcher), filter_(nullptr) {} + JsonDecoder(TagsMatcher &tagsMatcher, const FieldsSet *filter) noexcept : tagsMatcher_(tagsMatcher), filter_(filter) {} Error Decode(Payload &pl, WrSerializer &wrSer, const gason::JsonValue &v); void Decode(std::string_view json, CJsonBuilder &builder, const TagsPath &fieldPath); @@ -23,6 +23,7 @@ class JsonDecoder { TagsPath tagsPath_; const FieldsSet *filter_; int32_t arrayLevel_ = 0; + ScalarIndexesSetT objectScalarIndexes_; }; } // namespace reindexer diff --git a/cpp_src/core/cjson/msgpackdecoder.cc b/cpp_src/core/cjson/msgpackdecoder.cc index 47586b469..e29178282 100644 --- a/cpp_src/core/cjson/msgpackdecoder.cc +++ b/cpp_src/core/cjson/msgpackdecoder.cc @@ -1,5 +1,6 @@ #include "msgpackdecoder.h" +#include "core/cjson/cjsontools.h" #include "core/cjson/objtype.h" #include "core/cjson/tagsmatcher.h" #include "tools/flagguard.h" @@ -7,19 +8,15 @@ namespace reindexer { -MsgPackDecoder::MsgPackDecoder(TagsMatcher* tagsMatcher) : tm_(tagsMatcher) {} - template void MsgPackDecoder::setValue(Payload& pl, CJsonBuilder& builder, const T& value, int tagName) { - int field = tm_->tags2field(tagsPath_.data(), tagsPath_.size()); + int field = tm_.tags2field(tagsPath_.data(), tagsPath_.size()); if (field > 0) { - const auto& f = pl.Type().Field(field); - if (isInArray() && !f.IsArray()) { - throw Error(errLogic, "Error parsing msgpack field '%s' - got array, expected scalar %s", f.Name(), f.Type().Name()); - } + validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, pl.Type().Field(field), field, isInArray(), "msgpack"); Variant val(value); - pl.Set(field, val, true); builder.Ref(tagName, val, field); + pl.Set(field, std::move(val), true); + objectScalarIndexes_.set(field); } else { builder.Put(tagName, value); } @@ -35,13 +32,13 @@ int MsgPackDecoder::decodeKeyToTag(const msgpack_object_kv& obj) { using namespace std::string_view_literals; switch (obj.key.type) { case MSGPACK_OBJECT_BOOLEAN: - return tm_->name2tag(obj.key.via.boolean ? "true"sv : "false"sv, true); + return tm_.name2tag(obj.key.via.boolean ? "true"sv : "false"sv, true); case MSGPACK_OBJECT_POSITIVE_INTEGER: - return tm_->name2tag(std::to_string(obj.key.via.u64), true); + return tm_.name2tag(std::to_string(obj.key.via.u64), true); case MSGPACK_OBJECT_NEGATIVE_INTEGER: - return tm_->name2tag(std::to_string(obj.key.via.i64), true); + return tm_.name2tag(std::to_string(obj.key.via.i64), true); case MSGPACK_OBJECT_STR: - return tm_->name2tag(std::string_view(obj.key.via.str.ptr, obj.key.via.str.size), true); + return tm_.name2tag(std::string_view(obj.key.via.str.ptr, obj.key.via.str.size), true); case MSGPACK_OBJECT_FLOAT32: case MSGPACK_OBJECT_FLOAT64: case MSGPACK_OBJECT_NIL: @@ -49,9 +46,9 @@ int MsgPackDecoder::decodeKeyToTag(const msgpack_object_kv& obj) { case MSGPACK_OBJECT_MAP: case MSGPACK_OBJECT_BIN: case MSGPACK_OBJECT_EXT: - default: - throw Error(errParams, "Unsupported MsgPack map key type: %s(%d)", ToString(obj.key.type), obj.key.type); + break; } + throw Error(errParams, "Unsupported MsgPack map key type: %s(%d)", ToString(obj.key.type), obj.key.type); } void MsgPackDecoder::decode(Payload& pl, CJsonBuilder& builder, const msgpack_object& obj, int tagName) { @@ -91,8 +88,12 @@ void MsgPackDecoder::decode(Payload& pl, CJsonBuilder& builder, const msgpack_ob prevType = p->type; } } - int field = tm_->tags2field(tagsPath_.data(), tagsPath_.size()); + int field = tm_.tags2field(tagsPath_.data(), tagsPath_.size()); if (field > 0) { + auto& f = pl.Type().Field(field); + if rx_unlikely (!f.IsArray()) { + throw Error(errLogic, "Error parsing msgpack field '%s' - got array, expected scalar %s", f.Name(), f.Type().Name()); + } auto& array = builder.ArrayRef(tagName, field, count); iterateOverArray(begin, end, pl, array); } else { @@ -123,17 +124,18 @@ void MsgPackDecoder::decode(Payload& pl, CJsonBuilder& builder, const msgpack_ob Error MsgPackDecoder::Decode(std::string_view buf, Payload& pl, WrSerializer& wrser, size_t& offset) { try { + objectScalarIndexes_.reset(); tagsPath_.clear(); size_t baseOffset = offset; MsgPackValue data = parser_.Parse(buf, offset); - if (!data.p) return Error(errLogic, "Error unpacking msgpack data"); - if (data.p->type != MSGPACK_OBJECT_MAP) { + if rx_unlikely (!data.p) return Error(errLogic, "Error unpacking msgpack data"); + if rx_unlikely (data.p->type != MSGPACK_OBJECT_MAP) { std::string_view slice = buf.substr(baseOffset, 16); return Error(errNotValid, "Unexpected MsgPack value. Expected %s, but got %s(%d) at %d(~>\"%s\"...)", ToString(MSGPACK_OBJECT_MAP), ToString(data.p->type), data.p->type, baseOffset, slice); } - CJsonBuilder cjsonBuilder(wrser, ObjType::TypePlain, tm_, 0); + CJsonBuilder cjsonBuilder(wrser, ObjType::TypePlain, &tm_, 0); decode(pl, cjsonBuilder, *(data.p), 0); } catch (const Error& err) { return err; @@ -144,7 +146,7 @@ Error MsgPackDecoder::Decode(std::string_view buf, Payload& pl, WrSerializer& wr return Error(errNotValid, "Unexpected exception"); } - return errOK; + return Error(); } constexpr std::string_view ToString(msgpack_object_type type) { @@ -177,9 +179,8 @@ constexpr std::string_view ToString(msgpack_object_type type) { return "BIN"sv; case MSGPACK_OBJECT_EXT: return "EXT"sv; - default: - return "UNKNOWN_TYPE"sv; } + return "UNKNOWN_TYPE"sv; } } // namespace reindexer diff --git a/cpp_src/core/cjson/msgpackdecoder.h b/cpp_src/core/cjson/msgpackdecoder.h index 6cd3b552d..042b05263 100644 --- a/cpp_src/core/cjson/msgpackdecoder.h +++ b/cpp_src/core/cjson/msgpackdecoder.h @@ -14,7 +14,7 @@ class WrSerializer; class MsgPackDecoder { public: - explicit MsgPackDecoder(TagsMatcher* tagsMatcher); + explicit MsgPackDecoder(TagsMatcher& tagsMatcher) noexcept : tm_(tagsMatcher) {} Error Decode(std::string_view buf, Payload& pl, WrSerializer& wrser, size_t& offset); private: @@ -27,10 +27,11 @@ class MsgPackDecoder { void setValue(Payload& pl, CJsonBuilder& builder, const T& value, int tagName); bool isInArray() const noexcept { return arrayLevel_ > 0; } - TagsMatcher* tm_; + TagsMatcher& tm_; TagsPath tagsPath_; MsgPackParser parser_; int32_t arrayLevel_ = 0; + ScalarIndexesSetT objectScalarIndexes_; }; constexpr std::string_view ToString(msgpack_object_type type); diff --git a/cpp_src/core/cjson/protobufdecoder.cc b/cpp_src/core/cjson/protobufdecoder.cc index 7067d7a8b..2a1478cd0 100644 --- a/cpp_src/core/cjson/protobufdecoder.cc +++ b/cpp_src/core/cjson/protobufdecoder.cc @@ -6,8 +6,6 @@ namespace reindexer { -ArraysStorage::ArraysStorage(TagsMatcher& tm) : tm_(tm) {} - void ArraysStorage::UpdateArraySize(int tagName, int field) { GetArray(tagName, field); } CJsonBuilder& ArraysStorage::GetArray(int tagName, int field) { @@ -44,19 +42,21 @@ void ArraysStorage::onObjectBuilt(CJsonBuilder& parent) { indexes_.pop_back(); } -ProtobufDecoder::ProtobufDecoder(TagsMatcher& tagsMatcher, std::shared_ptr schema) - : tm_(tagsMatcher), schema_(std::move(schema)), arraysStorage_(tm_) {} - void ProtobufDecoder::setValue(Payload& pl, CJsonBuilder& builder, ProtobufValue item) { int field = tm_.tags2field(tagsPath_.data(), tagsPath_.size()); auto value = item.value.convert(item.itemType); if (field > 0) { - pl.Set(field, value, true); + const auto& f = pl.Type().Field(field); + if rx_unlikely (!f.IsArray() && objectScalarIndexes_.test(field)) { + throw Error(errLogic, "Non-array field '%s' [%d] from '%s' can only be encoded once.", f.Name(), field, pl.Type().Name()); + } if (item.isArray) { arraysStorage_.UpdateArraySize(item.tagName, field); } else { builder.Ref(item.tagName, value, field); } + pl.Set(field, std::move(value), true); + objectScalarIndexes_.set(field); } else { if (item.isArray) { auto& array = arraysStorage_.GetArray(item.tagName); @@ -73,6 +73,10 @@ Error ProtobufDecoder::decodeArray(Payload& pl, CJsonBuilder& builder, const Pro bool packed = item.IsOfPrimitiveType(); int field = tm_.tags2field(tagsPath_.data(), tagsPath_.size()); if (field > 0) { + auto& f = pl.Type().Field(field); + if rx_unlikely (!f.IsArray()) { + throw Error(errLogic, "Error parsing protobuf field '%s' - got array, expected scalar %s", f.Name(), f.Type().Name()); + } if (packed) { int count = 0; while (!parser.IsEof()) { @@ -91,7 +95,7 @@ Error ProtobufDecoder::decodeArray(Payload& pl, CJsonBuilder& builder, const Pro } } else { if (item.itemType.Is()) { - Error status{errOK}; + Error status; CJsonProtobufObjectBuilder obj(array, 0, arraysStorage_); while (status.ok() && !parser.IsEof()) { status = decode(pl, obj, parser.ReadValue()); @@ -101,7 +105,7 @@ Error ProtobufDecoder::decodeArray(Payload& pl, CJsonBuilder& builder, const Pro } } } - return errOK; + return Error(); } Error ProtobufDecoder::decode(Payload& pl, CJsonBuilder& builder, const ProtobufValue& item) { @@ -138,7 +142,7 @@ Error ProtobufDecoder::decode(Payload& pl, CJsonBuilder& builder, const Protobuf } Error ProtobufDecoder::decodeObject(Payload& pl, CJsonBuilder& builder, ProtobufObject& object) { - Error status{errOK}; + Error status; ProtobufParser parser(object); while (status.ok() && !parser.IsEof()) { status = decode(pl, builder, parser.ReadValue()); @@ -149,6 +153,7 @@ Error ProtobufDecoder::decodeObject(Payload& pl, CJsonBuilder& builder, Protobuf Error ProtobufDecoder::Decode(std::string_view buf, Payload& pl, WrSerializer& wrser) { try { tagsPath_.clear(); + objectScalarIndexes_.reset(); CJsonProtobufObjectBuilder cjsonBuilder(arraysStorage_, wrser, &tm_, 0); ProtobufObject object(buf, *schema_, tagsPath_, tm_); return decodeObject(pl, cjsonBuilder, object); diff --git a/cpp_src/core/cjson/protobufdecoder.h b/cpp_src/core/cjson/protobufdecoder.h index fce1f432c..7cba753d7 100644 --- a/cpp_src/core/cjson/protobufdecoder.h +++ b/cpp_src/core/cjson/protobufdecoder.h @@ -12,7 +12,7 @@ struct ProtobufObject; class ArraysStorage { public: - explicit ArraysStorage(TagsMatcher& tm); + explicit ArraysStorage(TagsMatcher& tm) noexcept : tm_(tm) {} ArraysStorage(const ArraysStorage&) = delete; ArraysStorage(ArraysStorage&&) = delete; ArraysStorage& operator=(const ArraysStorage&) = delete; @@ -68,7 +68,8 @@ class CJsonProtobufObjectBuilder { class ProtobufDecoder { public: - ProtobufDecoder(TagsMatcher& tagsMatcher, std::shared_ptr schema); + ProtobufDecoder(TagsMatcher& tagsMatcher, std::shared_ptr schema) noexcept + : tm_(tagsMatcher), schema_(std::move(schema)), arraysStorage_(tm_) {} ProtobufDecoder(const ProtobufDecoder&) = delete; ProtobufDecoder(ProtobufDecoder&&) = delete; ProtobufDecoder& operator=(const ProtobufDecoder&) = delete; @@ -86,6 +87,7 @@ class ProtobufDecoder { std::shared_ptr schema_; TagsPath tagsPath_; ArraysStorage arraysStorage_; + ScalarIndexesSetT objectScalarIndexes_; }; } // namespace reindexer diff --git a/cpp_src/core/cjson/tagsmatcher.h b/cpp_src/core/cjson/tagsmatcher.h index 584826c88..8bb8786b0 100644 --- a/cpp_src/core/cjson/tagsmatcher.h +++ b/cpp_src/core/cjson/tagsmatcher.h @@ -39,10 +39,10 @@ class TagsMatcher { auto res = impl_->path2indexedtag(jsonPath, ev); return res.empty() && canAdd ? impl_.clone()->path2indexedtag(jsonPath, ev, canAdd, updated_) : res; } - int version() const { return impl_->version(); } - size_t size() const { return impl_->size(); } - bool isUpdated() const { return updated_; } - uint32_t stateToken() const { return impl_->stateToken(); } + int version() const noexcept { return impl_->version(); } + size_t size() const noexcept { return impl_->size(); } + bool isUpdated() const noexcept { return updated_; } + uint32_t stateToken() const noexcept { return impl_->stateToken(); } void clear() { impl_.clone()->clear(); } void serialize(WrSerializer& ser) const { impl_->serialize(ser); } void deserialize(Serializer& ser) { @@ -53,8 +53,8 @@ class TagsMatcher { impl_.clone()->deserialize(ser, version, stateToken); impl_.clone()->buildTagsCache(updated_); } - void clearUpdated() { updated_ = false; } - void setUpdated() { updated_ = true; } + void clearUpdated() noexcept { updated_ = false; } + void setUpdated() noexcept { updated_ = true; } bool try_merge(const TagsMatcher& tm) { auto tmp = impl_; diff --git a/cpp_src/core/cjson/tagsmatcherimpl.h b/cpp_src/core/cjson/tagsmatcherimpl.h index c5754cd0f..e96109868 100644 --- a/cpp_src/core/cjson/tagsmatcherimpl.h +++ b/cpp_src/core/cjson/tagsmatcherimpl.h @@ -220,9 +220,9 @@ class TagsMatcherImpl { return true; } - size_t size() const { return tags2names_.size(); } - int version() const { return version_; } - int stateToken() const { return stateToken_; } + size_t size() const noexcept { return tags2names_.size(); } + int version() const noexcept { return version_; } + int stateToken() const noexcept { return stateToken_; } void clear() { names2tags_.clear(); diff --git a/cpp_src/core/comparatorimpl.h b/cpp_src/core/comparatorimpl.h index 844eb75be..41f0aa486 100644 --- a/cpp_src/core/comparatorimpl.h +++ b/cpp_src/core/comparatorimpl.h @@ -97,9 +97,9 @@ class ComparatorImpl { case CondLike: return false; case CondDWithin: - default: - abort(); + break; } + std::abort(); } bool Compare(CondType cond, T lhs) { bool ret = Compare2(cond, lhs); @@ -122,12 +122,19 @@ class ComparatorImpl { private: KeyValueType type() { - if constexpr (std::is_same_v) return KeyValueType::Int{}; - if constexpr (std::is_same_v) return KeyValueType::Bool{}; - if constexpr (std::is_same_v) return KeyValueType::Int64{}; - if constexpr (std::is_same_v) return KeyValueType::Double{}; - if constexpr (std::is_same_v) return KeyValueType::Uuid{}; - std::abort(); + if constexpr (std::is_same_v) + return KeyValueType::Int{}; + else if constexpr (std::is_same_v) + return KeyValueType::Bool{}; + else if constexpr (std::is_same_v) + return KeyValueType::Int64{}; + else if constexpr (std::is_same_v) + return KeyValueType::Double{}; + else if constexpr (std::is_same_v) + return KeyValueType::Uuid{}; + else { + static_assert(std::is_same_v, "Unknown KeyValueType"); + } } void addValue(CondType cond, T value) { @@ -203,9 +210,9 @@ class ComparatorImpl { case CondLike: return false; case CondDWithin: - default: - abort(); + break; } + std::abort(); } bool Compare(CondType cond, Uuid lhs) { bool ret = Compare2(cond, lhs); @@ -287,9 +294,9 @@ class ComparatorImpl { return matchLikePattern(std::string_view(lhs), rhs); } case CondDWithin: - default: - abort(); + break; } + std::abort(); } bool Compare(CondType cond, p_string lhs, const CollateOpts &collateOpts) { bool ret = Compare2(cond, lhs, collateOpts); @@ -393,9 +400,9 @@ class ComparatorImpl { case CondLike: return false; case CondDWithin: - default: - abort(); + break; } + std::abort(); } void ClearAllSetValues() { assertrx(allSetValuesSet_); diff --git a/cpp_src/core/expressiontree.md b/cpp_src/core/expressiontree.md index 1700e7782..772fdea75 100644 --- a/cpp_src/core/expressiontree.md +++ b/cpp_src/core/expressiontree.md @@ -19,9 +19,12 @@ class ExpressionTree; `ExpressionTree` does not support operator precedence. You can support it manually as it done in `QueryEntries` and `SelectIteratorContainer`, or by enclosing higher priority operators in brackets as it done in `SortExpression`. -Here do not used traditional way for constructing of trees with inheritance of nodes, allocations of separate nodes and holding of pointers to they. +Here is not used the traditional way for constructing trees with inheritance of nodes, allocations of separate nodes and holding pointers to them. `ExpressionTree` holds all nodes by value in a vector (`container_`) sequentially in type `Node` based on `variant`. -In order to support lazy copying `Node` can hold a reference to payload of another `Node` by using `ExpressionTree::Ref` type. !Warning! lazy copy should not live over the original one. +In order to support lazy copying `Node` can hold a reference to payload of another `Node` by using `ExpressionTree::Ref` type. + +**Warning**: The lazy copy node shall not live longer than the original one. + Subtree is stored in `container_` just behind its head (`SubTree`) which holds occupied space. For details see examples. This architecture allows to reduce count of allocations and virtual functions calls. @@ -187,21 +190,23 @@ It contains operation (value of `OperationType`) and a value of one of the types - `void Node::Append()` increments size of subexpression if it is head of subexpression, fails otherwise. - `void Node::Erase(size_t)` reduces size of subexpression if it is head of subexpression, fails otherwise. - ```c++ -template -void Node::ExecuteAppropriate(const std::function&... funcs); -template -void Node::ExecuteAppropriate(const std::function&... funcs) const; -``` -invoke appropriate functor if the `Node` holds value of one of `Args...` types or `Ref` where `T` is one of `Args...` types, no functor will be invoked otherwise. + template + void Node::ExecuteAppropriate(const std::function&... funcs); + template + void Node::ExecuteAppropriate(const std::function&... funcs) const; + ``` + + invokes appropriate functor if the `Node` holds value of one of `Args...` types or `Ref`, where `T` is one of `Args...` types, no functor will be invoked otherwise. + - ```c++ -template -R Node::CalculateAppropriate(const std::function& f, const std::function&... funcs) const; -``` -invokes appropriate functor depending on type of value is holded by `Node` and provides returned value. + template + R Node::CalculateAppropriate(const std::function& f, const std::function&... funcs) const; + ``` + invokes appropriate functor depending on type of value is held by `Node` and provides returned value. - `Node Node::MakeLazyCopy()&` -!Warning! the copy should not live over the origin. * returns copy of origin one if it is head of subexpression or holds value of `Ref` type. * returns new `Node` that holds `Ref` which references to payload of origin one if it holds `T` (one of `Ts...`). + > **Warning** the copy shall not live longer than the origin. - `Node Node::MakeDeepCopy() const &` * returns copy of origin one if it is head of subexpression or holds value of one of `Ts...` types. * returns new `Node` which holds copy of value that `Ref` references to if origin one holds value of `Ref` type. diff --git a/cpp_src/core/ft/ft_fast/dataprocessor.cc b/cpp_src/core/ft/ft_fast/dataprocessor.cc index 5a7f50b75..772b1c785 100644 --- a/cpp_src/core/ft/ft_fast/dataprocessor.cc +++ b/cpp_src/core/ft/ft_fast/dataprocessor.cc @@ -243,7 +243,7 @@ size_t DataProcessor::buildWordsMap(words_map &words_um) { } // Check and print potential stop words - if (rx_unlikely(holder_.cfg_->logLevel >= LogInfo)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogInfo) { WrSerializer out; for (auto &w : words_um) { if (w.second.vids_.size() > vdocs.size() / 5 || int64_t(w.second.vids_.size()) > holder_.cfg_->mergeLimit) { diff --git a/cpp_src/core/ft/ft_fast/selecter.cc b/cpp_src/core/ft/ft_fast/selecter.cc index e06e71989..b460ca929 100644 --- a/cpp_src/core/ft/ft_fast/selecter.cc +++ b/cpp_src/core/ft/ft_fast/selecter.cc @@ -60,7 +60,7 @@ void Selecter::prepareVariants(std::vector& variants, RV if (!term.opts.exact) { if (term.opts.op == OpNot && term.opts.suff) { // More strict match for negative (excluding) suffix terms - if (rx_unlikely(holder_.cfg_->logLevel >= LogTrace)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogTrace) { logPrintf(LogInfo, "Skipping stemming for '%s%s%s'", term.opts.suff ? "*" : "", tmpstr, term.opts.pref ? "*" : ""); } continue; @@ -75,7 +75,7 @@ void Selecter::prepareVariants(std::vector& variants, RV if (tmpstr != stemstr && !stemstr.empty()) { const auto charsCount = getUTF8StringCharactersCount(stemstr); if (charsCount <= kMaxStemSkipLen) { - if (rx_unlikely(holder_.cfg_->logLevel >= LogTrace)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogTrace) { logPrintf(LogInfo, "Skipping too short stemmer's term '%s%s*'", term.opts.suff && &v != &variantsUtf16[0] ? "*" : "", stemstr); } @@ -107,8 +107,8 @@ void Selecter::prepareVariants(std::vector& variants, RV // RX_NO_INLINE just for build test purpose. Do not expect any effect here template template -RX_NO_INLINE IDataHolder::MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, - const RdxContext& rdxCtx) { +RX_NO_INLINE IDataHolder::MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransaction, + FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) { FtSelectContext ctx; ctx.rawResults.reserve(dsl.size()); // STEP 2: Search dsl terms for each variant @@ -131,7 +131,7 @@ RX_NO_INLINE IDataHolder::MergeData Selecter::Process(FtDSLQuery&& dsl, ctx.lowRelVariants[j].rawResultIdx = ctx.rawResults.size() - 1; } - if (rx_unlikely(holder_.cfg_->logLevel >= LogInfo)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogInfo) { WrSerializer wrSer; wrSer << "variants: ["; for (auto& variant : ctx.variants) { @@ -182,7 +182,7 @@ RX_NO_INLINE IDataHolder::MergeData Selecter::Process(FtDSLQuery&& dsl, synCtx.rawResults.reserve(synDsl.dsl.size()); for (size_t i = 0; i < synDsl.dsl.size(); ++i) { prepareVariants(synCtx.variants, nullptr, i, holder_.cfg_->stemmers, synDsl.dsl, nullptr); - if (rx_unlikely(holder_.cfg_->logLevel >= LogInfo)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogInfo) { WrSerializer wrSer; for (auto& variant : synCtx.variants) { if (&variant != &*synCtx.variants.begin()) wrSer << ", "; @@ -237,7 +237,7 @@ void Selecter::processStepVariants(FtSelectContext& ctx, typename DataHo do { if (keyIt == suffixes.end()) break; if (vidsLimit <= vids) { - if (rx_unlikely(holder_.cfg_->logLevel >= LogInfo)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogInfo) { logPrintf(LogInfo, "Terminating suffix loop on limit (%d). Current variant is '%s%s%s'", initialLimit, variant.opts.suff ? "*" : "", variant.pattern, variant.opts.pref ? "*" : ""); } @@ -285,7 +285,7 @@ void Selecter::processStepVariants(FtSelectContext& ctx, typename DataHo ctx.totalORVids += vidsSize; } (*res.foundWords)[glbwordId] = std::make_pair(curRawResultIdx, res.size() - 1); - if (rx_unlikely(holder_.cfg_->logLevel >= LogTrace)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogTrace) { logPrintf(LogInfo, " matched %s '%s' of word '%s' (variant '%s'), %d vids, %d%%", suffixLen ? "suffix" : "prefix", keyIt->first, word, variant.pattern, holder_.getWordById(glbwordId).vids_.size(), proc); } @@ -297,7 +297,7 @@ void Selecter::processStepVariants(FtSelectContext& ctx, typename DataHo ++skipped; } } while ((keyIt++).lcp() >= int(tmpstr.length())); - if (rx_unlikely(holder_.cfg_->logLevel >= LogInfo)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogInfo) { std::string limitString; if (vidsLimit <= vids) { limitString = fmt::sprintf(". Lookup terminated by VIDs limit(%d)", initialLimit); @@ -348,7 +348,7 @@ void Selecter::processLowRelVariants(FtSelectContext& ctx, const FtMerge if constexpr (kVariantsWithDifLength) { if (variant.GetLenCached() != lastVariantLen) { if (unsigned(targetORLimit) <= ctx.totalORVids) { - if (rx_unlikely(holder_.cfg_->logLevel >= LogTrace)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogTrace) { logPrintf(LogInfo, "Terminating on target OR limit. Current vairiant is '%s%s%s'", variant.opts.suff ? "*" : "", variant.pattern, variant.opts.pref ? "*" : ""); } @@ -359,7 +359,7 @@ void Selecter::processLowRelVariants(FtSelectContext& ctx, const FtMerge } else { (void)lastVariantLen; } - if (rx_unlikely(holder_.cfg_->logLevel >= LogTrace)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogTrace) { logPrintf(LogInfo, "Handling '%s%s%s' as variant with low relevancy", variant.opts.suff ? "*" : "", variant.pattern, variant.opts.pref ? "*" : ""); } @@ -715,7 +715,7 @@ void Selecter::mergeIteration(TextSearchResults& rawRes, index_t rawResI if (!termRank) { continue; } - if (rx_unlikely(holder_.cfg_->logLevel >= LogTrace)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogTrace) { logPrintf(LogInfo, "Pattern %s, idf %f, termLenBoost %f", r.pattern, idf, rawRes.term.opts.termLenBoost); } @@ -838,7 +838,7 @@ void Selecter::mergeIterationGroup(TextSearchResults& rawRes, index_t ra auto [termRank, field] = calcTermRank(rawRes, idf, relid, r.proc_); if (!termRank) continue; - if (rx_unlikely(holder_.cfg_->logLevel >= LogTrace)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogTrace) { logPrintf(LogInfo, "Pattern %s, idf %f, termLenBoost %f", r.pattern, idf, rawRes.term.opts.termLenBoost); } @@ -1036,7 +1036,8 @@ void Selecter::TyposHandler::operator()(std::vector& } const uint8_t wordLength = step.suffixes_.word_len_at(wordIdSfx); - const int proc = std::max(holder.cfg_->rankingConfig.typo - + const int proc = + std::max(holder.cfg_->rankingConfig.typo - tcount * holder.cfg_->rankingConfig.typoPenalty / std::max((wordLength - tcount) / 3, BaseFTConfig::BaseRankingConfig::kMinProcAfterPenalty), 1); @@ -1058,7 +1059,7 @@ void Selecter::TyposHandler::operator()(std::vector& if (dontUseMaxTyposForBoth_ && level == 1 && typo.size() != patternSize) return; } }); - if (rx_unlikely(holder.cfg_->logLevel >= LogInfo)) { + if rx_unlikely (holder.cfg_->logLevel >= LogInfo) { logPrintf(LogInfo, "Lookup typos, matched %d typos, with %d vids, skiped %d", matched, vids, skiped); } } @@ -1069,7 +1070,7 @@ RX_ALWAYS_INLINE unsigned uabs(int a) { return unsigned(std::abs(a)); } template template void Selecter::TyposHandler::logTraceF(int level, const char* fmt, Args&&... args) { - if (rx_unlikely(logLevel_ >= LogTrace)) { + if rx_unlikely (logLevel_ >= LogTrace) { logPrintf(level, fmt, std::forward(args)...); } } @@ -1316,7 +1317,7 @@ typename IDataHolder::MergeData Selecter::mergeResults(std::vectorlogLevel >= LogInfo)) { + if rx_unlikely (holder_.cfg_->logLevel >= LogInfo) { logPrintf(LogInfo, "Complex merge (%d patterns): out %d vids", rawResults.size(), merged.size()); } diff --git a/cpp_src/core/ft/ft_fuzzy/prefilter/prefilter.h b/cpp_src/core/ft/ft_fuzzy/prefilter/prefilter.h index 6c86a770a..c06ca9081 100644 --- a/cpp_src/core/ft/ft_fuzzy/prefilter/prefilter.h +++ b/cpp_src/core/ft/ft_fuzzy/prefilter/prefilter.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include #include #include diff --git a/cpp_src/core/index/index.cc b/cpp_src/core/index/index.cc index 50a0d4dac..552033fc2 100644 --- a/cpp_src/core/index/index.cc +++ b/cpp_src/core/index/index.cc @@ -46,6 +46,7 @@ std::unique_ptr Index::New(const IndexDef& idef, PayloadType payloadType, case IndexInt64Store: case IndexDoubleStore: case IndexBool: + case IndexUuidStore: return IndexStore_New(idef, std::move(payloadType), fields); case IndexFastFT: case IndexCompositeFastFT: @@ -59,9 +60,8 @@ std::unique_ptr Index::New(const IndexDef& idef, PayloadType payloadType, return IndexRTree_New(idef, std::move(payloadType), fields); case IndexUuidHash: return IndexUuid_New(idef, std::move(payloadType), fields); - default: - throw Error(errParams, "Ivalid index type %d for index '%s'", idef.Type(), idef.name_); } + throw Error(errParams, "Ivalid index type %d for index '%s'", idef.Type(), idef.name_); } template diff --git a/cpp_src/core/index/indexordered.cc b/cpp_src/core/index/indexordered.cc index 08859b62c..828030b1b 100644 --- a/cpp_src/core/index/indexordered.cc +++ b/cpp_src/core/index/indexordered.cc @@ -235,9 +235,10 @@ static std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType case IndexTtl: case IndexRTree: case IndexUuidHash: - default: - abort(); + case IndexUuidStore: + break; } + std::abort(); } // NOLINTBEGIN(*cplusplus.NewDeleteLeaks) diff --git a/cpp_src/core/index/indexstore.cc b/cpp_src/core/index/indexstore.cc index 43654c377..27fe6239f 100644 --- a/cpp_src/core/index/indexstore.cc +++ b/cpp_src/core/index/indexstore.cc @@ -106,6 +106,41 @@ SelectKeyResults IndexStore::SelectKey(const VariantArray &keys, CondType con if (condition == CondAny && !this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct) throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes"); + // TODO: it may be necessary to remove or change this switch after QueryEntry refactoring + switch (condition) { + case CondAny: + if (!this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct) { + throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes"); + } + break; + case CondEmpty: + if (!this->opts_.IsArray() && !this->opts_.IsSparse()) { + throw Error(errParams, "The 'is NULL' condition is suported only by 'sparse' or 'array' indexes"); + } + break; + case CondAllSet: + case CondSet: + case CondEq: + break; + case CondRange: + case CondDWithin: + if (keys.size() != 2) { + throw Error(errParams, "For condition %s required exactly 2 arguments, but provided %d", CondTypeToStr(condition), + keys.size()); + } + break; + case CondLt: + case CondLe: + case CondGt: + case CondGe: + case CondLike: + if (keys.size() != 1) { + throw Error(errParams, "For condition %s required exactly 1 argument, but provided %d", CondTypeToStr(condition), + keys.size()); + } + break; + } + res.comparators_.push_back(Comparator(condition, KeyType(), keys, opts_.IsArray(), sopts.distinct, payloadType_, fields_, idx_data.size() ? idx_data.data() : nullptr, opts_.collateOpts_)); return SelectKeyResults(std::move(res)); @@ -160,6 +195,8 @@ std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType payloadT return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; case IndexStrStore: return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + case IndexUuidStore: + return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; case IndexStrHash: case IndexStrBTree: case IndexIntBTree: @@ -176,9 +213,9 @@ std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType payloadT case IndexTtl: case IndexRTree: case IndexUuidHash: - default: - abort(); + break; } + std::abort(); } template class IndexStore; diff --git a/cpp_src/core/index/indexstore.h b/cpp_src/core/index/indexstore.h index 3a25b48b6..832c11a4f 100644 --- a/cpp_src/core/index/indexstore.h +++ b/cpp_src/core/index/indexstore.h @@ -27,6 +27,7 @@ class IndexStore : public Index { void Dump(std::ostream &os, std::string_view step = " ", std::string_view offset = "") const override { dump(os, step, offset); } virtual void AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue &) override; virtual bool IsDestroyPartSupported() const noexcept override { return true; } + virtual bool IsUuid() const noexcept override final { return std::is_same_v; } template struct HasAddTask : std::false_type {}; diff --git a/cpp_src/core/index/indextext/fastindextext.cc b/cpp_src/core/index/indextext/fastindextext.cc index 90e050e9c..36db2dc49 100644 --- a/cpp_src/core/index/indextext/fastindextext.cc +++ b/cpp_src/core/index/indextext/fastindextext.cc @@ -46,7 +46,7 @@ void FastIndexText::initHolder(FtFastConfig &cfg) { template Variant FastIndexText::Upsert(const Variant &key, IdType id, bool &clearCache) { - if (rx_unlikely(key.Type().Is())) { + if rx_unlikely (key.Type().Is()) { if (this->empty_ids_.Unsorted().Add(id, IdSet::Auto, 0)) { this->isBuilt_ = false; } @@ -78,7 +78,7 @@ Variant FastIndexText::Upsert(const Variant &key, IdType id, bool &clearCache template void FastIndexText::Delete(const Variant &key, IdType id, StringsHolder &strHolder, bool &clearCache) { int delcnt = 0; - if (rx_unlikely(key.Type().Is())) { + if rx_unlikely (key.Type().Is()) { delcnt = this->empty_ids_.Unsorted().Erase(id); assertrx(delcnt); this->isBuilt_ = false; @@ -132,10 +132,10 @@ IndexMemStat FastIndexText::GetMemStat(const RdxContext &ctx) { template IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery &&dsl, bool inTransaction, FtMergeStatuses &&statuses, FtUseExternStatuses useExternSt, const RdxContext &rdxCtx) { - fctx->GetData()->extraWordSymbols_ = this->GetConfig()->extraWordSymbols; + fctx->GetData()->extraWordSymbols_ = this->getConfig()->extraWordSymbols; fctx->GetData()->isWordPositions_ = true; - auto mergeData = this->holder_->Select(std::move(dsl), this->fields_.size(), fctx->NeedArea(), GetConfig()->maxAreasInDoc, + auto mergeData = this->holder_->Select(std::move(dsl), this->fields_.size(), fctx->NeedArea(), getConfig()->maxAreasInDoc, inTransaction, std::move(statuses.statuses), useExternSt, rdxCtx); // convert vids(uniq documents id) to ids (real ids) IdSet::Ptr mergedIds = make_intrusive>(); @@ -146,7 +146,7 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery &&dsl, bool inTr } int cnt = 0; const double scalingFactor = mergeData.maxRank > 255 ? 255.0 / mergeData.maxRank : 1.0; - int minRelevancy = GetConfig()->minRelevancy * 100 * scalingFactor; + const int minRelevancy = getConfig()->minRelevancy * 100 * scalingFactor; size_t releventDocs = 0; for (auto &vid : mergeData) { auto &vdoc = holder.vdocs_[vid.id]; @@ -155,38 +155,49 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery &&dsl, bool inTr } vid.proc *= scalingFactor; if (vid.proc <= minRelevancy) break; + + assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); cnt += vdoc.keyEntry->Sorted(0).size(); ++releventDocs; } mergedIds->reserve(cnt); fctx->Reserve(cnt); - for (size_t i = 0; i < releventDocs; ++i) { - auto &vid = mergeData[i]; - auto &vdoc = holder.vdocs_[vid.id]; - if (!vdoc.keyEntry) { - continue; + if (!fctx->NeedArea()) { + if (useExternSt == FtUseExternStatuses::No) { + appendMergedIds(mergeData, releventDocs, + [&fctx, &mergedIds](IdSetRef::iterator ebegin, IdSetRef::iterator eend, const IDataHolder::MergeInfo &vid) { + fctx->Add(ebegin, eend, vid.proc); + mergedIds->Append(ebegin, eend, IdSet::Unordered); + }); + } else { + appendMergedIds( + mergeData, releventDocs, + [&fctx, &mergedIds, &statuses](IdSetRef::iterator ebegin, IdSetRef::iterator eend, const IDataHolder::MergeInfo &vid) { + fctx->Add(ebegin, eend, vid.proc, statuses.rowIds); + mergedIds->Append(ebegin, eend, statuses.rowIds, IdSet::Unordered); + }); } - assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); - auto ebegin = vdoc.keyEntry->Sorted(0).begin(); - auto eend = vdoc.keyEntry->Sorted(0).end(); + } else { if (useExternSt == FtUseExternStatuses::No) { - if (vid.areaIndex == std::numeric_limits::max()) { - fctx->Add(ebegin, eend, vid.proc); - } else { - fctx->Add(ebegin, eend, vid.proc, std::move(mergeData.vectorAreas[vid.areaIndex])); - } - mergedIds->Append(ebegin, eend, IdSet::Unordered); + appendMergedIds( + mergeData, releventDocs, + [&fctx, &mergedIds, &mergeData](IdSetRef::iterator ebegin, IdSetRef::iterator eend, const IDataHolder::MergeInfo &vid) { + assertrx_throw(vid.areaIndex != std::numeric_limits::max()); + fctx->Add(ebegin, eend, vid.proc, std::move(mergeData.vectorAreas[vid.areaIndex])); + mergedIds->Append(ebegin, eend, IdSet::Unordered); + }); } else { - if (vid.areaIndex == std::numeric_limits::max()) { - fctx->Add(ebegin, eend, vid.proc, statuses.rowIds); - } else { - fctx->Add(ebegin, eend, vid.proc, statuses.rowIds, std::move(mergeData.vectorAreas[vid.areaIndex])); - } - mergedIds->Append(ebegin, eend, statuses.rowIds, IdSet::Unordered); + appendMergedIds(mergeData, releventDocs, + [&fctx, &mergedIds, &mergeData, &statuses](IdSetRef::iterator ebegin, IdSetRef::iterator eend, + const IDataHolder::MergeInfo &vid) { + assertrx_throw(vid.areaIndex != std::numeric_limits::max()); + fctx->Add(ebegin, eend, vid.proc, statuses.rowIds, std::move(mergeData.vectorAreas[vid.areaIndex])); + mergedIds->Append(ebegin, eend, statuses.rowIds, IdSet::Unordered); + }); } } - if (rx_unlikely(GetConfig()->logLevel >= LogInfo)) { + if rx_unlikely (getConfig()->logLevel >= LogInfo) { logPrintf(LogInfo, "Total merge out: %d ids", mergedIds->size()); std::string str; @@ -238,7 +249,7 @@ void FastIndexText::commitFulltextImpl() { } } } - if (rx_unlikely(GetConfig()->logLevel >= LogInfo)) { + if rx_unlikely (getConfig()->logLevel >= LogInfo) { auto tm2 = high_resolution_clock::now(); logPrintf(LogInfo, "FastIndexText::Commit elapsed %d ms total [ build vdocs %d ms, process data %d ms ]", duration_cast(tm2 - tm0).count(), duration_cast(tm1 - tm0).count(), @@ -290,7 +301,7 @@ void FastIndexText::buildVdocs(Container &data) { vdocs.push_back({doc->second.get(), {}, {}}); #endif - if (rx_unlikely(GetConfig()->logLevel <= LogInfo)) { + if rx_unlikely (getConfig()->logLevel <= LogInfo) { for (auto &f : vdocsTexts.back()) this->holder_->szCnt += f.first.length(); } } @@ -300,9 +311,18 @@ void FastIndexText::buildVdocs(Container &data) { } template -FtFastConfig *FastIndexText::GetConfig() const { - return dynamic_cast(this->cfg_.get()); +template +RX_ALWAYS_INLINE void FastIndexText::appendMergedIds(IDataHolder::MergeData &mergeData, size_t releventDocs, F &&appender) { + auto &holder = *this->holder_; + for (size_t i = 0; i < releventDocs; ++i) { + auto &vid = mergeData[i]; + auto &vdoc = holder.vdocs_[vid.id]; + if (vdoc.keyEntry) { + appender(vdoc.keyEntry->Sorted(0).begin(), vdoc.keyEntry->Sorted(0).end(), vid); + } + } } + template void FastIndexText::initConfig(const FtFastConfig *cfg) { if (cfg) { @@ -311,7 +331,7 @@ void FastIndexText::initConfig(const FtFastConfig *cfg) { this->cfg_.reset(new FtFastConfig(this->ftFields_.size())); this->cfg_->parse(this->opts_.config, this->ftFields_); } - initHolder(*GetConfig()); // -V522 + initHolder(*getConfig()); // -V522 this->holder_->synonyms_->SetConfig(this->cfg_.get()); } @@ -322,9 +342,9 @@ bool eq_c(Container &c1, Container &c2) { template void FastIndexText::SetOpts(const IndexOpts &opts) { - auto oldCfg = *GetConfig(); + auto oldCfg = *getConfig(); IndexText::SetOpts(opts); - auto &newCfg = *GetConfig(); + auto &newCfg = *getConfig(); if (!eq_c(oldCfg.stopWords, newCfg.stopWords) || oldCfg.stemmers != newCfg.stemmers || oldCfg.maxTypoLen != newCfg.maxTypoLen || oldCfg.enableNumbersSearch != newCfg.enableNumbersSearch || oldCfg.extraWordSymbols != newCfg.extraWordSymbols || @@ -378,9 +398,10 @@ std::unique_ptr FastIndexText_New(const IndexDef &idef, PayloadType paylo case IndexTtl: case IndexRTree: case IndexUuidHash: - default: - abort(); + case IndexUuidStore: + break; } + std::abort(); } } // namespace reindexer diff --git a/cpp_src/core/index/indextext/fastindextext.h b/cpp_src/core/index/indextext/fastindextext.h index 99280ffce..5a6ebb50d 100644 --- a/cpp_src/core/index/indextext/fastindextext.h +++ b/cpp_src/core/index/indextext/fastindextext.h @@ -17,7 +17,7 @@ class FastIndexText : public IndexText { using ref_type = typename IndexUnordered::ref_type; FastIndexText(const FastIndexText& other) : Base(other) { - initConfig(other.GetConfig()); + initConfig(other.getConfig()); for (auto& idx : this->idx_map) idx.second.SetVDocID(FtKeyEntryData::ndoc); this->CommitFulltext(); } @@ -38,16 +38,18 @@ class FastIndexText : public IndexText { &holder_->rowId2Vdoc_}; } reindexer::FtPreselectT FtPreselect(const RdxContext& rdxCtx) override final; - bool EnablePreselectBeforeFt() const override final { return GetConfig()->enablePreselectBeforeFt; } + bool EnablePreselectBeforeFt() const override final { return getConfig()->enablePreselectBeforeFt; } protected: void commitFulltextImpl() override final; - FtFastConfig* GetConfig() const; + FtFastConfig* getConfig() const noexcept { return dynamic_cast(this->cfg_.get()); } void initConfig(const FtFastConfig* = nullptr); void initHolder(FtFastConfig&); - template void buildVdocs(Data& data); + template + void appendMergedIds(IDataHolder::MergeData& merged, size_t releventDocs, F&& appender); + std::unique_ptr holder_; }; diff --git a/cpp_src/core/index/indextext/fuzzyindextext.cc b/cpp_src/core/index/indextext/fuzzyindextext.cc index 2c509aa4f..339ac1252 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.cc +++ b/cpp_src/core/index/indextext/fuzzyindextext.cc @@ -24,7 +24,7 @@ IdSet::Ptr FuzzyIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inT size_t counter = 0; for (auto it = result.data_->begin(); it != result.data_->end(); ++it, ++counter) { it->proc_ *= coof; - if (it->proc_ < GetConfig()->minOkProc) continue; + if (it->proc_ < getConfig()->minOkProc) continue; assertrx(it->id_ < this->vdocs_.size()); const auto& id_set = this->vdocs_[it->id_].keyEntry->Sorted(0); fctx->Add(id_set.begin(), id_set.end(), it->proc_); @@ -54,10 +54,7 @@ void FuzzyIndexText::commitFulltextImpl() { engine_.Commit(); this->isBuilt_ = true; } -template -FtFuzzyConfig* FuzzyIndexText::GetConfig() const { - return dynamic_cast(this->cfg_.get()); -} + template void FuzzyIndexText::CreateConfig(const FtFuzzyConfig* cfg) { if (cfg) { @@ -94,9 +91,10 @@ std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType payl case IndexTtl: case IndexRTree: case IndexUuidHash: - default: - abort(); + case IndexUuidStore: + break; } + std::abort(); } } // namespace reindexer diff --git a/cpp_src/core/index/indextext/fuzzyindextext.h b/cpp_src/core/index/indextext/fuzzyindextext.h index d3bc6d50b..d1a94780a 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.h +++ b/cpp_src/core/index/indextext/fuzzyindextext.h @@ -13,7 +13,7 @@ class FuzzyIndexText : public IndexText { using Base = IndexText; public: - FuzzyIndexText(const FuzzyIndexText& other) : Base(other) { CreateConfig(other.GetConfig()); } + FuzzyIndexText(const FuzzyIndexText& other) : Base(other) { CreateConfig(other.getConfig()); } FuzzyIndexText(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) : Base(idef, std::move(payloadType), fields) { CreateConfig(); @@ -42,7 +42,7 @@ class FuzzyIndexText : public IndexText { protected: void commitFulltextImpl() override final; - FtFuzzyConfig* GetConfig() const; + FtFuzzyConfig* getConfig() const noexcept { return dynamic_cast(this->cfg_.get()); } void CreateConfig(const FtFuzzyConfig* cfg = nullptr); search_engine::SearchEngine engine_; diff --git a/cpp_src/core/index/indextext/indextext.cc b/cpp_src/core/index/indextext/indextext.cc index 7ca8e5f8b..4a8a646db 100644 --- a/cpp_src/core/index/indextext/indextext.cc +++ b/cpp_src/core/index/indextext/indextext.cc @@ -28,10 +28,10 @@ void IndexText::initSearchers() { ftFields_.emplace(this->payloadType_->Field(fieldIdx).Name(), i); } } - if (rx_unlikely(ftFields_.size() != this->fields_.size())) { + if rx_unlikely (ftFields_.size() != this->fields_.size()) { throw Error(errParams, "Composite fulltext index '%s' contains duplicated fields", this->name_); } - if (rx_unlikely(ftFields_.size() > kMaxFtCompositeFields)) { + if rx_unlikely (ftFields_.size() > kMaxFtCompositeFields) { throw Error(errParams, "Unable to create composite fulltext '%s' index with %d fields. Fileds count limit is %d", this->name_, ftFields_.size(), kMaxFtCompositeFields); } @@ -58,7 +58,7 @@ void IndexText::SetOpts(const IndexOpts &opts) { template FtCtx::Ptr IndexText::prepareFtCtx(const BaseFunctionCtx::Ptr &ctx) { FtCtx::Ptr ftctx = reindexer::reinterpret_pointer_cast(ctx); - if (rx_unlikely(!ftctx)) { + if rx_unlikely (!ftctx) { throw Error(errParams, "Full text index (%s) may not be used without context", Index::Name()); } ftctx->PrepareAreas(ftFields_, this->name_); @@ -83,7 +83,7 @@ template SelectKeyResults IndexText::SelectKey(const VariantArray &keys, CondType condition, SortType, Index::SelectOpts opts, const BaseFunctionCtx::Ptr &ctx, const RdxContext &rdxCtx) { const auto indexWard(rdxCtx.BeforeIndexWork()); - if (rx_unlikely(keys.size() < 1 || (condition != CondEq && condition != CondSet))) { + if rx_unlikely (keys.size() < 1 || (condition != CondEq && condition != CondSet)) { throw Error(errParams, "Full text index (%s) support only EQ or SET condition with 1 or 2 parameter", Index::Name()); } @@ -107,7 +107,7 @@ SelectKeyResults IndexText::SelectKey(const VariantArray &keys, CondType cond template SelectKeyResults IndexText::resultFromCache(const VariantArray &keys, FtIdSetCache::Iterator &&it, FtCtx::Ptr &ftctx) { - if (rx_unlikely(cfg_->logLevel >= LogInfo)) { + if rx_unlikely (cfg_->logLevel >= LogInfo) { logPrintf(LogInfo, "Get search results for '%s' in '%s' from cache", keys[0].As(), this->payloadType_ ? this->payloadType_->Name() : ""); } @@ -124,7 +124,7 @@ template SelectKeyResults IndexText::doSelectKey(const VariantArray &keys, const std::optional &ckey, FtMergeStatuses &&mergeStatuses, FtUseExternStatuses useExternSt, bool inTransaction, FtCtx::Ptr ftctx, const RdxContext &rdxCtx) { - if (rx_unlikely(cfg_->logLevel >= LogInfo)) { + if rx_unlikely (cfg_->logLevel >= LogInfo) { logPrintf(LogInfo, "Searching for '%s' in '%s' %s", keys[0].As(), this->payloadType_ ? this->payloadType_->Name() : "", ckey ? "(will cache)" : ""); } @@ -170,7 +170,7 @@ template SelectKeyResults IndexText::SelectKey(const VariantArray &keys, CondType condition, Index::SelectOpts opts, const BaseFunctionCtx::Ptr &ctx, FtPreselectT &&preselect, const RdxContext &rdxCtx) { const auto indexWard(rdxCtx.BeforeIndexWork()); - if (rx_unlikely(keys.size() < 1 || (condition != CondEq && condition != CondSet))) { + if rx_unlikely (keys.size() < 1 || (condition != CondEq && condition != CondSet)) { throw Error(errParams, "Full text index (%s) support only EQ or SET condition with 1 or 2 parameter", Index::Name()); } return doSelectKey(keys, std::nullopt, std::move(preselect), FtUseExternStatuses::Yes, opts.inTransaction, prepareFtCtx(ctx), rdxCtx); diff --git a/cpp_src/core/index/indexunordered.cc b/cpp_src/core/index/indexunordered.cc index 6b1d5869e..b311fe25f 100644 --- a/cpp_src/core/index/indexunordered.cc +++ b/cpp_src/core/index/indexunordered.cc @@ -462,9 +462,10 @@ static std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadTy case IndexTtl: case IndexRTree: case IndexUuidHash: - default: - abort(); + case IndexUuidStore: + break; } + std::abort(); } // NOLINTBEGIN(*cplusplus.NewDeleteLeaks) diff --git a/cpp_src/core/index/uuid_index.h b/cpp_src/core/index/uuid_index.h index 0863eee22..1352c3f43 100644 --- a/cpp_src/core/index/uuid_index.h +++ b/cpp_src/core/index/uuid_index.h @@ -13,7 +13,6 @@ class UuidIndex : public IndexUnordered std::unique_ptr Clone() const override { return std::unique_ptr{new UuidIndex{*this}}; } using Base::Upsert; void Upsert(VariantArray& result, const VariantArray& keys, IdType id, bool& clearCache) override; // TODO delete this after #1353 - bool IsUuid() const noexcept override { return true; } }; std::unique_ptr IndexUuid_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); diff --git a/cpp_src/core/indexdef.cc b/cpp_src/core/indexdef.cc index 59c07e177..8195a2fac 100644 --- a/cpp_src/core/indexdef.cc +++ b/cpp_src/core/indexdef.cc @@ -68,6 +68,7 @@ static const std::unordered_map, std::equal {IndexFuzzyFT, {"string"s, "fuzzytext"s, condsText(), CapFullText}}, {IndexRTree, {"point"s, "rtree"s, condsGeom(), 0}}, {IndexUuidHash, {"uuid"s, "hash"s, condsUsual(), CapSortable}}, + {IndexUuidStore, {"uuid"s, "-"s, condsUsual(), CapSortable}}, }; // clang-format on return data; @@ -122,13 +123,14 @@ bool IndexDef::IsEqual(const IndexDef &other, bool skipConfig) const { } IndexType IndexDef::Type() const { - std::string iType = indexType_; + using namespace std::string_view_literals; + std::string_view iType = indexType_; if (iType == "") { - if (fieldType_ == "double") { + if (fieldType_ == "double"sv) { iType = "tree"; - } else if (fieldType_ == "bool") { + } else if (fieldType_ == "bool"sv) { iType = "-"; - } else if (fieldType_ == "point") { + } else if (fieldType_ == "point"sv) { iType = "rtree"; } else { iType = "hash"; @@ -156,7 +158,8 @@ const std::vector &IndexDef::Conditions() const { bool isSortable(IndexType type) { return availableIndexes().at(type).caps & CapSortable; } bool isStore(IndexType type) noexcept { - return type == IndexIntStore || type == IndexInt64Store || type == IndexStrStore || type == IndexDoubleStore || type == IndexBool; + return type == IndexIntStore || type == IndexInt64Store || type == IndexStrStore || type == IndexDoubleStore || type == IndexBool || + type == IndexUuidStore; } std::string IndexDef::getCollateMode() const { return availableCollates().at(opts_.GetCollateMode()); } diff --git a/cpp_src/core/item.cc b/cpp_src/core/item.cc index 949bf6cfe..7840e7853 100644 --- a/cpp_src/core/item.cc +++ b/cpp_src/core/item.cc @@ -8,9 +8,6 @@ namespace reindexer { -Item::FieldRef::FieldRef(int field, ItemImpl *itemImpl) : itemImpl_(itemImpl), field_(field) {} -Item::FieldRef::FieldRef(std::string_view jsonPath, ItemImpl *itemImpl) : itemImpl_(itemImpl), jsonPath_(jsonPath), field_(-1) {} - Item &Item::operator=(Item &&other) noexcept { if (&other != this) { if (impl_) { @@ -52,12 +49,12 @@ Item::FieldRef::operator Variant() const { } Item::FieldRef::operator VariantArray() const { - VariantArray kr; - if (field_ >= 0) + if (field_ >= 0) { + VariantArray kr; itemImpl_->GetPayload().Get(field_, kr); - else - kr = itemImpl_->GetValueByJSONPath(jsonPath_); - return kr; + return kr; + } + return itemImpl_->GetValueByJSONPath(jsonPath_); } Item::FieldRef &Item::FieldRef::operator=(Variant kr) { @@ -122,10 +119,10 @@ Item::~Item() { auto ns = impl_->GetNamespace(); if (ns) { ns->ToPool(impl_); - impl_ = nullptr; + } else { + delete impl_; } } - delete impl_; } Error Item::FromJSON(std::string_view slice, char **endp, bool pkOnly) &noexcept { @@ -153,25 +150,24 @@ Error Item::GetMsgPack(WrSerializer &wrser) &noexcept { RETURN_RESULT_NOEXCEPT(i Error Item::GetProtobuf(WrSerializer &wrser) &noexcept { RETURN_RESULT_NOEXCEPT(impl_->GetProtobuf(wrser)); } int Item::NumFields() const { return impl_->Type().NumFields(); } + Item::FieldRef Item::operator[](int field) const { - assertrx(field >= 0 && field < impl_->Type().NumFields()); + if (rx_unlikely(field < 0 || field >= impl_->Type().NumFields())) { + throw Error(errLogic, "Item::operator[] requires indexed field. Values range: [0; %d]", impl_->Type().NumFields()); + } return FieldRef(field, impl_); } Item::FieldRef Item::operator[](std::string_view name) const { int field = 0; - if (impl_->Type().FieldByName(name, field)) { - return FieldRef(field, impl_); - } else { - return FieldRef(name, impl_); - } + return (impl_->Type().FieldByName(name, field)) ? FieldRef(field, impl_) : FieldRef(name, impl_); } int Item::GetFieldTag(std::string_view name) const { return impl_->NameTag(name); } FieldsSet Item::PkFields() const { return impl_->PkFields(); } void Item::SetPrecepts(const std::vector &precepts) & { impl_->SetPrecepts(precepts); } -bool Item::IsTagsUpdated() { return impl_->tagsMatcher().isUpdated(); } -int Item::GetStateToken() { return impl_->tagsMatcher().stateToken(); } +bool Item::IsTagsUpdated() const noexcept { return impl_->tagsMatcher().isUpdated(); } +int Item::GetStateToken() const noexcept { return impl_->tagsMatcher().stateToken(); } Item &Item::Unsafe(bool enable) &noexcept { impl_->Unsafe(enable); diff --git a/cpp_src/core/item.h b/cpp_src/core/item.h index b0bee1b49..1a81e836b 100644 --- a/cpp_src/core/item.h +++ b/cpp_src/core/item.h @@ -109,8 +109,8 @@ class Item { FieldRef &operator=(const VariantArray &krs); private: - FieldRef(int field, ItemImpl *itemImpl); - FieldRef(std::string_view jsonPath, ItemImpl *itemImpl); + FieldRef(int field, ItemImpl *itemImpl) noexcept : itemImpl_(itemImpl), field_(field) {} + FieldRef(std::string_view jsonPath, ItemImpl *itemImpl) noexcept : itemImpl_(itemImpl), jsonPath_(jsonPath), field_(-1) {} ItemImpl *itemImpl_; std::string_view jsonPath_; int field_; @@ -190,12 +190,12 @@ class Item { void SetPrecepts(const std::vector &precepts) &; /// Check was names tags updated while modify operation /// @return true: tags was updated. - [[nodiscard]] bool IsTagsUpdated(); + [[nodiscard]] bool IsTagsUpdated() const noexcept; /// Get state token /// @return Current state token - [[nodiscard]] int GetStateToken(); + [[nodiscard]] int GetStateToken() const noexcept; /// Check is item valid. If is not valid, then any futher operations with item will raise nullptr dereference - [[nodiscard]] bool operator!() const { return impl_ == nullptr; } + [[nodiscard]] bool operator!() const noexcept { return impl_ == nullptr; } /// Enable Unsafe Mode
. /// USE WITH CAUTION. In unsafe mode most of Item methods will not store strings and slices, passed from/to application.
/// The advantage of unsafe mode is speed. It does not call extra memory allocation from heap and copying data.
diff --git a/cpp_src/core/itemimpl.cc b/cpp_src/core/itemimpl.cc index e80161acc..eb8555094 100644 --- a/cpp_src/core/itemimpl.cc +++ b/cpp_src/core/itemimpl.cc @@ -86,7 +86,7 @@ void ItemImpl::GetField(int field, VariantArray &values) { GetPayload().Get(fiel Error ItemImpl::FromMsgPack(std::string_view buf, size_t &offset) { Payload pl = GetPayload(); if (!msgPackDecoder_) { - msgPackDecoder_.reset(new MsgPackDecoder(&tagsMatcher_)); + msgPackDecoder_.reset(new MsgPackDecoder(tagsMatcher_)); } ser_.Reset(); @@ -123,7 +123,7 @@ Error ItemImpl::GetMsgPack(WrSerializer &wrser) { MsgPackBuilder msgpackBuilder(wrser, &tagsLengths, &startTag, ObjType::TypePlain, &tagsMatcher_); msgpackEncoder.Encode(pl, msgpackBuilder); - return errOK; + return Error(); } Error ItemImpl::GetProtobuf(WrSerializer &wrser) { @@ -132,7 +132,7 @@ Error ItemImpl::GetProtobuf(WrSerializer &wrser) { ProtobufBuilder protobufBuilder(&wrser, ObjType::TypePlain, schema_.get(), &tagsMatcher_); ProtobufEncoder protobufEncoder(&tagsMatcher_); protobufEncoder.Encode(pl, protobufBuilder); - return errOK; + return Error(); } // Construct item from compressed json diff --git a/cpp_src/core/itemmodifier.cc b/cpp_src/core/itemmodifier.cc index 80746f090..fb2cfa52c 100644 --- a/cpp_src/core/itemmodifier.cc +++ b/cpp_src/core/itemmodifier.cc @@ -114,8 +114,7 @@ ItemModifier::ItemModifier(const std::vector &updateEntries, Namesp } } -void ItemModifier::Modify(IdType itemId, const NsContext &ctx) { - assertrx(ctx.noLock); +void ItemModifier::Modify(IdType itemId) { PayloadValue &pv = ns_.items_[itemId]; Payload pl(ns_.payloadType_, pv); pv.Clone(pl.RealSize()); @@ -140,34 +139,36 @@ void ItemModifier::Modify(IdType itemId, const NsContext &ctx) { } if (field.details().Mode() == FieldModeSetJson || !field.isIndex()) { - modifyCJSON(pv, itemId, field, values, ctx); + modifyCJSON(pv, itemId, field, values); } else { - modifyField(itemId, field, pl, values, ctx); + modifyField(itemId, field, pl, values); } } ns_.markUpdated(false); } -void ItemModifier::modifyCJSON(PayloadValue &pv, IdType id, FieldData &field, VariantArray &values, const NsContext &ctx) { +void ItemModifier::modifyCJSON(PayloadValue &pv, IdType id, FieldData &field, VariantArray &values) { PayloadValue &plData = ns_.items_[id]; - Payload pl(ns_.payloadType_, plData); + Payload pl(*ns_.payloadType_.get(), plData); VariantArray cjsonKref; pl.Get(0, cjsonKref); cjsonCache_.Reset(); - if (cjsonKref.size() > 0) { - Variant v = cjsonKref.front(); - if (v.Type().Is()) { - cjsonCache_.Assign(std::string_view(p_string(v))); - } + + const Variant &v = cjsonKref.front(); + if (v.Type().Is()) { + cjsonCache_.Assign(std::string_view(p_string(v))); } ItemImpl itemimpl(ns_.payloadType_, pv, ns_.tagsMatcher_); itemimpl.ModifyField(field.tagspath(), values, field.details().Mode()); - Item item = ns_.NewItem(ctx); + Item item = ns_.newItem(); Error err = item.FromCJSON(itemimpl.GetCJSON(true)); - if (!err.ok()) throw err; + if (!err.ok()) { + pl.Set(0, cjsonKref); + throw err; + } item.setID(id); ItemImpl *impl = item.impl_; ns_.setFieldsBasedOnPrecepts(impl); @@ -176,7 +177,7 @@ void ItemModifier::modifyCJSON(PayloadValue &pv, IdType id, FieldData &field, Va Payload plNew = impl->GetPayload(); plData.Clone(pl.RealSize()); - auto strHolder = ns_.StrHolder(ctx); + auto strHolder = ns_.strHolder(); auto indexesCacheCleaner{ns_.GetIndexesCacheCleaner()}; h_vector needUpdateCompIndexes(ns_.indexes_.compositeIndexesSize(), false); for (int i = ns_.indexes_.firstCompositePos(); i < ns_.indexes_.totalSize(); ++i) { @@ -268,7 +269,7 @@ void ItemModifier::modifyCJSON(PayloadValue &pv, IdType id, FieldData &field, Va impl->RealValue() = pv; } -void ItemModifier::modifyField(IdType itemId, FieldData &field, Payload &pl, VariantArray &values, const NsContext &ctx) { +void ItemModifier::modifyField(IdType itemId, FieldData &field, Payload &pl, VariantArray &values) { Index &index = *(ns_.indexes_[field.index()]); if (field.isIndex() && !index.Opts().IsSparse() && field.details().Mode() == FieldModeDrop /*&& !(field.arrayIndex() != IndexValueType::NotSet || field.tagspath().back().IsArrayNode())*/) { // TODO #1218 allow to drop array fields @@ -284,7 +285,7 @@ void ItemModifier::modifyField(IdType itemId, FieldData &field, Payload &pl, Var for (const Variant &key : values) key.EnsureUTF8(); } - auto strHolder = ns_.StrHolder(ctx); + auto strHolder = ns_.strHolder(); auto indexesCacheCleaner{ns_.GetIndexesCacheCleaner()}; h_vector needUpdateCompIndexes(ns_.indexes_.compositeIndexesSize(), false); const auto firstCompositePos = ns_.indexes_.firstCompositePos(); @@ -326,7 +327,7 @@ void ItemModifier::modifyField(IdType itemId, FieldData &field, Payload &pl, Var try { if (field.isIndex()) { - modifyIndexValues(itemId, field, values, pl, ctx); + modifyIndexValues(itemId, field, values, pl); } if (index.Opts().IsSparse() || index.Opts().IsArray() || index.KeyType().Is() || !field.isIndex()) { @@ -360,12 +361,12 @@ void ItemModifier::modifyField(IdType itemId, FieldData &field, Payload &pl, Var insertItemIntoCompositeIndexes(); } -void ItemModifier::modifyIndexValues(IdType itemId, const FieldData &field, VariantArray &values, Payload &pl, const NsContext &ctx) { +void ItemModifier::modifyIndexValues(IdType itemId, const FieldData &field, VariantArray &values, Payload &pl) { Index &index = *(ns_.indexes_[field.index()]); if (values.IsNullValue() && !index.Opts().IsArray()) { throw Error(errParams, "Non-array index fields cannot be set to null!"); } - auto strHolder = ns_.StrHolder(ctx); + auto strHolder = ns_.strHolder(); auto indexesCacheCleaner{ns_.GetIndexesCacheCleaner()}; bool updateArrayPart = field.arrayIndex() >= 0; bool isForAllItems = false; diff --git a/cpp_src/core/itemmodifier.h b/cpp_src/core/itemmodifier.h index e3cede700..e970f993f 100644 --- a/cpp_src/core/itemmodifier.h +++ b/cpp_src/core/itemmodifier.h @@ -18,7 +18,7 @@ class ItemModifier { ItemModifier(ItemModifier &&) = delete; ItemModifier &operator=(ItemModifier &&) = delete; - void Modify(IdType itemId, const NsContext &ctx); + void Modify(IdType itemId); private: struct FieldData { @@ -61,9 +61,9 @@ class ItemModifier { std::string_view cjson_; }; - void modifyField(IdType itemId, FieldData &field, Payload &pl, VariantArray &values, const NsContext &); - void modifyCJSON(PayloadValue &pv, IdType itemId, FieldData &field, VariantArray &values, const NsContext &); - void modifyIndexValues(IdType itemId, const FieldData &field, VariantArray &values, Payload &pl, const NsContext &); + void modifyField(IdType itemId, FieldData &field, Payload &pl, VariantArray &values); + void modifyCJSON(PayloadValue &pv, IdType itemId, FieldData &field, VariantArray &values); + void modifyIndexValues(IdType itemId, const FieldData &field, VariantArray &values, Payload &pl); NamespaceImpl &ns_; const std::vector &updateEntries_; diff --git a/cpp_src/core/keyvalue/geometry.h b/cpp_src/core/keyvalue/geometry.h index e23e811fa..af779eba8 100644 --- a/cpp_src/core/keyvalue/geometry.h +++ b/cpp_src/core/keyvalue/geometry.h @@ -20,10 +20,10 @@ class Point { private: void validate(double v, std::string_view name) { - if (rx_unlikely(std::isinf(v))) { + if rx_unlikely (std::isinf(v)) { throwInfError(name); } - if (rx_unlikely(std::isnan(v))) { + if rx_unlikely (std::isnan(v)) { throwNanError(name); } } diff --git a/cpp_src/core/keyvalue/uuid.cc b/cpp_src/core/keyvalue/uuid.cc index 6103bc26d..5a549252b 100644 --- a/cpp_src/core/keyvalue/uuid.cc +++ b/cpp_src/core/keyvalue/uuid.cc @@ -6,76 +6,190 @@ namespace reindexer { Uuid::Uuid(std::string_view str) : data_{0, 0} { const auto err = tryParse(str, data_); - if (!err.ok()) { + if (rx_unlikely(!err.ok())) { throw err; } } +#ifdef GET_NUM +static_assert(false, "GET_NUM is already defined"); +#endif + +#define GET_NUM(i) \ + num = hexCharToNum[static_cast(str[i])]; \ + if (rx_unlikely(num > 15)) { \ + if (str[i] == '-') { \ + return Error(errNotValid, "Invalid UUID format: '%s'", str); \ + } else { \ + return Error(errNotValid, "UUID cannot contain char '%c': '%s'", str[i], str); \ + } \ + } + Error Uuid::tryParse(std::string_view str, uint64_t (&data)[2]) noexcept { - unsigned i = 0; - for (char ch : str) { - if (i >= 32 && ch != '-') { + static constexpr uint64_t hexCharToNum[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 1, 2, 3, + 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}; + static constexpr size_t kHexDigitsCount = 32; + uint64_t num; + switch (str.size()) { + case kHexDigitsCount: + GET_NUM(0) + data[0] = num << 60; + GET_NUM(1) + data[0] |= num << 56; + GET_NUM(2) + data[0] |= num << 52; + GET_NUM(3) + data[0] |= num << 48; + GET_NUM(4) + data[0] |= num << 44; + GET_NUM(5) + data[0] |= num << 40; + GET_NUM(6) + data[0] |= num << 36; + GET_NUM(7) + data[0] |= num << 32; + GET_NUM(8) + data[0] |= num << 28; + GET_NUM(9) + data[0] |= num << 24; + GET_NUM(10) + data[0] |= num << 20; + GET_NUM(11) + data[0] |= num << 16; + GET_NUM(12) + data[0] |= num << 12; + GET_NUM(13) + data[0] |= num << 8; + GET_NUM(14) + data[0] |= num << 4; + GET_NUM(15) + data[0] |= num; + GET_NUM(16) + data[1] = num << 60; + GET_NUM(17) + data[1] |= num << 56; + GET_NUM(18) + data[1] |= num << 52; + GET_NUM(19) + data[1] |= num << 48; + GET_NUM(20) + data[1] |= num << 44; + GET_NUM(21) + data[1] |= num << 40; + GET_NUM(22) + data[1] |= num << 36; + GET_NUM(23) + data[1] |= num << 32; + GET_NUM(24) + data[1] |= num << 28; + GET_NUM(25) + data[1] |= num << 24; + GET_NUM(26) + data[1] |= num << 20; + GET_NUM(27) + data[1] |= num << 16; + GET_NUM(28) + data[1] |= num << 12; + GET_NUM(29) + data[1] |= num << 8; + GET_NUM(30) + data[1] |= num << 4; + GET_NUM(31) + data[1] |= num; + break; + case kStrFormLen: + if (rx_unlikely(str[8] != '-' || str[13] != '-' || str[18] != '-' || str[23] != '-')) { + return Error(errNotValid, "Invalid UUID format: '%s'", str); + } + GET_NUM(0) + data[0] = num << 60; + GET_NUM(1) + data[0] |= num << 56; + GET_NUM(2) + data[0] |= num << 52; + GET_NUM(3) + data[0] |= num << 48; + GET_NUM(4) + data[0] |= num << 44; + GET_NUM(5) + data[0] |= num << 40; + GET_NUM(6) + data[0] |= num << 36; + GET_NUM(7) + data[0] |= num << 32; + GET_NUM(9) + data[0] |= num << 28; + GET_NUM(10) + data[0] |= num << 24; + GET_NUM(11) + data[0] |= num << 20; + GET_NUM(12) + data[0] |= num << 16; + GET_NUM(14) + data[0] |= num << 12; + GET_NUM(15) + data[0] |= num << 8; + GET_NUM(16) + data[0] |= num << 4; + GET_NUM(17) + data[0] |= num; + GET_NUM(19) + data[1] = num << 60; + GET_NUM(20) + data[1] |= num << 56; + GET_NUM(21) + data[1] |= num << 52; + GET_NUM(22) + data[1] |= num << 48; + GET_NUM(24) + data[1] |= num << 44; + GET_NUM(25) + data[1] |= num << 40; + GET_NUM(26) + data[1] |= num << 36; + GET_NUM(27) + data[1] |= num << 32; + GET_NUM(28) + data[1] |= num << 28; + GET_NUM(29) + data[1] |= num << 24; + GET_NUM(30) + data[1] |= num << 20; + GET_NUM(31) + data[1] |= num << 16; + GET_NUM(32) + data[1] |= num << 12; + GET_NUM(33) + data[1] |= num << 8; + GET_NUM(34) + data[1] |= num << 4; + GET_NUM(35) + data[1] |= num; + break; + default: return Error(errNotValid, "UUID should consist of 32 hexadecimal digits: '%s'", str); - } - uint64_t value; - switch (ch) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - value = ch - '0'; - break; - case 'a': - case 'A': - value = 0xA; - break; - case 'b': - case 'B': - value = 0xB; - break; - case 'c': - case 'C': - value = 0xC; - break; - case 'd': - case 'D': - value = 0xD; - break; - case 'e': - case 'E': - value = 0xE; - break; - case 'f': - case 'F': - value = 0xF; - break; - case '-': - continue; - default: - return Error(errNotValid, "UUID cannot contain char '%c': '%s'", ch, str); - } - data[i / 16] = (data[i / 16] << 4) | value; - ++i; } - if (i != 32) { - return Error(errNotValid, "UUID should consist of 32 hexadecimal digits: '%s'", str); - } - if ((data[0] != 0 || data[1] != 0) && (data[1] >> 63) == 0) { + if (rx_unlikely((data[0] != 0 || data[1] != 0) && (data[1] >> 63) == 0)) { return Error(errNotValid, "Variant 0 of UUID is unsupported: '%s'", str); } return {}; } +#undef GET_NUM + std::optional Uuid::TryParse(std::string_view str) noexcept { Uuid ret; const auto err = tryParse(str, ret.data_); - if (err.ok()) { + if (rx_likely(err.ok())) { return ret; } else { return std::nullopt; @@ -91,21 +205,43 @@ Uuid::operator std::string() const { void Uuid::PutToStr(span str) const noexcept { assertrx(str.size() >= kStrFormLen); - static constexpr char ch[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - for (size_t i = 0, j = 0; i < kStrFormLen; ++i) { - switch (i) { - case 8: - case 13: - case 18: - case 23: - str[i] = '-'; - break; - default: - str[i] = ch[(data_[j / 16] >> ((15 - j % 16) * 4)) & 0xF]; - ++j; - break; - } - } + static constexpr char hexChars[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + str[0] = hexChars[(data_[0] >> 60) & 0xF]; + str[1] = hexChars[(data_[0] >> 56) & 0xF]; + str[2] = hexChars[(data_[0] >> 52) & 0xF]; + str[3] = hexChars[(data_[0] >> 48) & 0xF]; + str[4] = hexChars[(data_[0] >> 44) & 0xF]; + str[5] = hexChars[(data_[0] >> 40) & 0xF]; + str[6] = hexChars[(data_[0] >> 36) & 0xF]; + str[7] = hexChars[(data_[0] >> 32) & 0xF]; + str[8] = '-'; + str[9] = hexChars[(data_[0] >> 28) & 0xF]; + str[10] = hexChars[(data_[0] >> 24) & 0xF]; + str[11] = hexChars[(data_[0] >> 20) & 0xF]; + str[12] = hexChars[(data_[0] >> 16) & 0xF]; + str[13] = '-'; + str[14] = hexChars[(data_[0] >> 12) & 0xF]; + str[15] = hexChars[(data_[0] >> 8) & 0xF]; + str[16] = hexChars[(data_[0] >> 4) & 0xF]; + str[17] = hexChars[data_[0] & 0xF]; + str[18] = '-'; + str[19] = hexChars[(data_[1] >> 60) & 0xF]; + str[20] = hexChars[(data_[1] >> 56) & 0xF]; + str[21] = hexChars[(data_[1] >> 52) & 0xF]; + str[22] = hexChars[(data_[1] >> 48) & 0xF]; + str[23] = '-'; + str[24] = hexChars[(data_[1] >> 44) & 0xF]; + str[25] = hexChars[(data_[1] >> 40) & 0xF]; + str[26] = hexChars[(data_[1] >> 36) & 0xF]; + str[27] = hexChars[(data_[1] >> 32) & 0xF]; + str[28] = hexChars[(data_[1] >> 28) & 0xF]; + str[29] = hexChars[(data_[1] >> 24) & 0xF]; + str[30] = hexChars[(data_[1] >> 20) & 0xF]; + str[31] = hexChars[(data_[1] >> 16) & 0xF]; + str[32] = hexChars[(data_[1] >> 12) & 0xF]; + str[33] = hexChars[(data_[1] >> 8) & 0xF]; + str[34] = hexChars[(data_[1] >> 4) & 0xF]; + str[35] = hexChars[data_[1] & 0xF]; } } // namespace reindexer diff --git a/cpp_src/core/keyvalue/variant.cc b/cpp_src/core/keyvalue/variant.cc index 3b3947c60..71d80b12e 100644 --- a/cpp_src/core/keyvalue/variant.cc +++ b/cpp_src/core/keyvalue/variant.cc @@ -171,10 +171,11 @@ std::string Variant::As() const { [&](KeyValueType::Int64) { return std::to_string(variant_.value_int64); }, [&](KeyValueType::Double) { return std::to_string(variant_.value_double); }, [&](KeyValueType::String) { - if (this->operator p_string().type() == p_string::tagCxxstr || this->operator p_string().type() == p_string::tagKeyString) { - return *(this->operator p_string().getCxxstr()); + const auto pstr = this->operator p_string(); + if (pstr.type() == p_string::tagCxxstr || pstr.type() == p_string::tagKeyString) { + return *(pstr.getCxxstr()); } - return this->operator p_string().toString(); + return pstr.toString(); }, [&](KeyValueType::Null) { return "null"s; }, [&](KeyValueType::Composite) { return std::string(); }, [&](KeyValueType::Tuple) { @@ -359,7 +360,7 @@ int Variant::Compare(const Variant &other, const CollateOpts &collateOpts) const : (variant_.value_double > other.variant_.value_double) ? 1 : -1; }, - [&](KeyValueType::Tuple) { return getCompositeValues() == other.getCompositeValues() ? 0 : 1; }, + [&](KeyValueType::Tuple) -> int { throw Error(errParams, "KeyValueType::Tuple comparison is not implemented"); }, [&](KeyValueType::String) { return collateCompare(this->operator p_string(), other.operator p_string(), collateOpts); }, [&](KeyValueType::Uuid) { return Uuid{*this}.Compare(Uuid{other}); }, [](KeyValueType::Null) -> int { @@ -539,7 +540,8 @@ size_t Variant::Hash() const noexcept { void Variant::EnsureUTF8() const { if (!isUuid() && variant_.type.Is()) { - if (!utf8::is_valid(operator p_string().data(), operator p_string().data() + operator p_string().size())) { + const auto pstr = this->operator p_string(); + if (!utf8::is_valid(pstr.data(), pstr.data() + pstr.size())) { throw Error(errParams, "Invalid UTF8 string passed to index with CollateUTF8 mode"); } } diff --git a/cpp_src/core/keyvalue/variant.h b/cpp_src/core/keyvalue/variant.h index 9df1d2c29..a4296aacb 100644 --- a/cpp_src/core/keyvalue/variant.h +++ b/cpp_src/core/keyvalue/variant.h @@ -126,9 +126,8 @@ class Variant { KeyValueType Type() const noexcept { if (isUuid()) { return KeyValueType::Uuid{}; - } else { - return variant_.type; } + return variant_.type; } Variant &convert(KeyValueType type, const PayloadType * = nullptr, const FieldsSet * = nullptr) &; @@ -217,7 +216,14 @@ class VariantArray : public h_vector { VariantArray() noexcept = default; explicit VariantArray(Point) noexcept; explicit operator Point() const; - void MarkArray() noexcept { isArrayValue = true; } + VariantArray &MarkArray(bool v = true) &noexcept { + isArrayValue = v; + return *this; + } + VariantArray &&MarkArray(bool v = true) &&noexcept { + isArrayValue = v; + return std::move(*this); + } void MarkObject() noexcept { isObjectValue = true; } using h_vector::h_vector; using h_vector::operator==; @@ -242,6 +248,11 @@ class VariantArray : public h_vector { static VariantArray Create(Ts &&...vs) { return VariantArray{Variant{std::forward(vs)}...}; } + void Clear() noexcept { + clear(); + isArrayValue = false; + isObjectValue = false; + } private: bool isArrayValue = false; diff --git a/cpp_src/core/lrucache.cc b/cpp_src/core/lrucache.cc index 7f04bbf32..ecfc29ac8 100644 --- a/cpp_src/core/lrucache.cc +++ b/cpp_src/core/lrucache.cc @@ -13,7 +13,7 @@ const int kMaxHitCountToCache = 1024; template typename LRUCache::Iterator LRUCache::Get(const K &key) { - if (rx_unlikely(cacheSizeLimit_ == 0)) return Iterator(); + if rx_unlikely (cacheSizeLimit_ == 0) return Iterator(); std::lock_guard lk(lock_); @@ -21,7 +21,7 @@ typename LRUCache::Iterator LRUCache::Get( if (emplaced) { totalCacheSize_ += kElemSizeOverhead + sizeof(Entry) + key.Size(); it->second.lruPos = lru_.insert(lru_.end(), &it->first); - if (rx_unlikely(!eraseLRU())) { + if rx_unlikely (!eraseLRU()) { return Iterator(); } } else if (std::next(it->second.lruPos) != lru_.end()) { @@ -41,7 +41,7 @@ typename LRUCache::Iterator LRUCache::Get( template void LRUCache::Put(const K &key, V &&v) { - if (rx_unlikely(cacheSizeLimit_ == 0)) return; + if rx_unlikely (cacheSizeLimit_ == 0) return; std::lock_guard lk(lock_); auto it = items_.find(key); @@ -56,7 +56,7 @@ void LRUCache::Put(const K &key, V &&v) { eraseLRU(); - if (rx_unlikely(putCount_ * 16 > getCount_ && eraseCount_)) { + if rx_unlikely (putCount_ * 16 > getCount_ && eraseCount_) { logPrintf(LogWarning, "IdSetCache::eraseLRU () cache invalidates too fast eraseCount=%d,putCount=%d,getCount=%d", eraseCount_, putCount_, eraseCount_); eraseCount_ = 0; @@ -74,7 +74,7 @@ RX_ALWAYS_INLINE bool LRUCache::eraseLRU() { // just to save us if totalCacheSize_ >0 and lru is empty // someone can make bad key or val with wrong size // TODO: Probably we should remove this logic, since there is no access to sizes outside of the lrucache - if (rx_unlikely(lru_.empty())) { + if rx_unlikely (lru_.empty()) { clearAll(); logPrintf(LogError, "IdSetCache::eraseLRU () Cache restarted because wrong cache size totalCacheSize_=%d", totalCacheSize_); return false; @@ -84,7 +84,7 @@ RX_ALWAYS_INLINE bool LRUCache::eraseLRU() { const size_t oldSize = sizeof(Entry) + kElemSizeOverhead + mIt->first.Size() + mIt->second.val.Size(); - if (rx_unlikely(oldSize > totalCacheSize_)) { + if rx_unlikely (oldSize > totalCacheSize_) { clearAll(); logPrintf(LogError, "IdSetCache::eraseLRU () Cache restarted because wrong cache size totalCacheSize_=%d,oldSize=%d", totalCacheSize_, oldSize); diff --git a/cpp_src/core/lrucache.h b/cpp_src/core/lrucache.h index 8b07b7e1e..e81fa741f 100644 --- a/cpp_src/core/lrucache.h +++ b/cpp_src/core/lrucache.h @@ -87,7 +87,7 @@ class LRUCache { auto mIt = items_.find(**it); assertrx(mIt != items_.end()); const size_t oldSize = sizeof(Entry) + kElemSizeOverhead + mIt->first.Size() + mIt->second.val.Size(); - if (rx_unlikely(oldSize > totalCacheSize_)) { + if rx_unlikely (oldSize > totalCacheSize_) { clearAll(); return; } diff --git a/cpp_src/core/namespace/bgnamespacedeleter.h b/cpp_src/core/namespace/bgnamespacedeleter.h new file mode 100644 index 000000000..77e0cfd3b --- /dev/null +++ b/cpp_src/core/namespace/bgnamespacedeleter.h @@ -0,0 +1,32 @@ +#pragma once + +#include "namespaceimpl.h" + +namespace reindexer { + +class BackgroundNamespaceDeleter { +public: + void Add(NamespaceImpl::Ptr ns) { + std::lock_guard lck(mtx_); + namespaces_.emplace_back(std::move(ns)); + } + void DeleteUnique() noexcept { + std::unique_lock lck(mtx_); + for (auto it = namespaces_.begin(); it != namespaces_.end();) { + if (it->unique()) { + lck.unlock(); + it->reset(); + lck.lock(); + it = namespaces_.erase(it); + } else { + ++it; + } + } + } + +private: + std::mutex mtx_; + std::list namespaces_; +}; + +} diff --git a/cpp_src/core/namespace/namespace.cc b/cpp_src/core/namespace/namespace.cc index cfb3d53f1..3bc79b243 100644 --- a/cpp_src/core/namespace/namespace.cc +++ b/cpp_src/core/namespace/namespace.cc @@ -17,7 +17,7 @@ void Namespace::CommitTransaction(Transaction& tx, QueryResults& result, const R } bool wasCopied = false; // NOLINT(*deadcode.DeadStores) auto params = longTxLoggingParams_.load(std::memory_order_relaxed); - QueryStatCalculator statCalculator(long_actions::Logger{tx, params, wasCopied}, params.thresholdUs >= 0); + QueryStatCalculator statCalculator(long_actions::MakeLogger(tx, params, wasCopied), params.thresholdUs >= 0); PerfStatCalculatorMT txCommitCalc(commitStatsCounter_, enablePerfCounters); if (needNamespaceCopy(nsl, tx)) { @@ -40,10 +40,12 @@ void Namespace::CommitTransaction(Transaction& tx, QueryResults& result, const R cg.Reset(); nsCopy_.reset(new NamespaceImpl(*nsl, storageLock)); nsCopyCalc.HitManualy(); - nsCopy_->CommitTransaction(tx, result, NsContext(ctx).NoLock(), statCalculator); + NsContext nsCtx(ctx); + nsCtx.CopiedNsRequest(); + nsCopy_->CommitTransaction(tx, result, nsCtx, statCalculator); if (nsCopy_->lastUpdateTime_) { nsCopy_->lastUpdateTime_ -= nsCopy_->config_.optimizationTimeout * 2; - nsCopy_->optimizeIndexes(NsContext(ctx).NoLock()); + nsCopy_->optimizeIndexes(nsCtx); nsCopy_->warmupFtIndexes(); } try { @@ -65,6 +67,7 @@ void Namespace::CommitTransaction(Transaction& tx, QueryResults& result, const R hasCopy_.store(false, std::memory_order_release); throw; } + bgDeleter_.Add(std::move(nsl)); nsl = ns_; lck.unlock(); statCalculator.LogFlushDuration(nsl->storage_, &AsyncStorage::TryForceFlush); diff --git a/cpp_src/core/namespace/namespace.h b/cpp_src/core/namespace/namespace.h index 8430b1e70..71864740d 100644 --- a/cpp_src/core/namespace/namespace.h +++ b/cpp_src/core/namespace/namespace.h @@ -2,9 +2,10 @@ #include #include +#include "bgnamespacedeleter.h" +#include "core/querystat.h" #include "core/txstats.h" #include "estl/shared_mutex.h" -#include "namespaceimpl.h" namespace reindexer { @@ -12,9 +13,10 @@ namespace reindexer { class Namespace { public: - Namespace(const std::string &name, UpdatesObservers &observers) : ns_(std::make_shared(name, observers)) {} - Namespace(NamespaceImpl::Ptr ns) : ns_(std::move(ns)) {} - typedef shared_ptr Ptr; + using Ptr = shared_ptr; + + Namespace(const std::string &name, UpdatesObservers &observers, BackgroundNamespaceDeleter &bgDeleter) + : ns_(make_intrusive(name, observers)), bgDeleter_(bgDeleter) {} void CommitTransaction(Transaction &tx, QueryResults &result, const RdxContext &ctx); std::string GetName(const RdxContext &ctx) const { return handleInvalidation(NamespaceImpl::GetName)(ctx); } @@ -34,25 +36,33 @@ class Namespace { void SetSchema(std::string_view schema, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::SetSchema)(schema, ctx); } std::string GetSchema(int format, const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetSchema)(format, ctx); } std::shared_ptr GetSchemaPtr(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetSchemaPtr)(ctx); } - void Insert(Item &item, const NsContext &ctx) { handleInvalidation(NamespaceImpl::Insert)(item, ctx); } - void Insert(Item &item, QueryResults &qr, const NsContext &ctx) { nsFuncWrapper<&NamespaceImpl::Insert>(item, qr, ctx); } - void Update(Item &item, const NsContext &ctx) { - nsFuncWrapper(item, ctx); + void Insert(Item &item, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Insert)(item, ctx); } + void Insert(Item &item, QueryResults &qr, const RdxContext &ctx) { + nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeInsert>(item, qr, ctx); + } + void Update(Item &item, const RdxContext &ctx) { + nsFuncWrapper(item, ctx); + } + void Update(Item &item, QueryResults &qr, const RdxContext &ctx) { + nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeUpdate>(item, qr, ctx); + } + void Update(const Query &query, QueryResults &result, const RdxContext &ctx) { + nsFuncWrapper<&NamespaceImpl::doUpdate, QueryType::QueryUpdate>(query, result, ctx); } - void Update(Item &item, QueryResults &qr, const NsContext &ctx) { nsFuncWrapper<&NamespaceImpl::Update>(item, qr, ctx); } - void Update(const Query &query, QueryResults &result, const NsContext &ctx) { - nsFuncWrapper<&NamespaceImpl::Update>(query, result, ctx); + void Upsert(Item &item, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Upsert)(item, ctx); } + void Upsert(Item &item, QueryResults &qr, const RdxContext &ctx) { + nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeUpsert>(item, qr, ctx); } - void Upsert(Item &item, const NsContext &ctx) { handleInvalidation(NamespaceImpl::Upsert)(item, ctx); } - void Upsert(Item &item, QueryResults &qr, const NsContext &ctx) { nsFuncWrapper<&NamespaceImpl::Upsert>(item, qr, ctx); } - void Delete(Item &item, const NsContext &ctx) { - nsFuncWrapper(item, ctx); + void Delete(Item &item, const RdxContext &ctx) { + nsFuncWrapper(item, ctx); } - void Delete(Item &item, QueryResults &qr, const NsContext &ctx) { nsFuncWrapper<&NamespaceImpl::Delete>(item, qr, ctx); } - void Delete(const Query &query, QueryResults &result, const NsContext &ctx) { - nsFuncWrapper<&NamespaceImpl::Delete>(query, result, ctx); + void Delete(Item &item, QueryResults &qr, const RdxContext &ctx) { + nsFuncWrapper<&NamespaceImpl::modifyItem, ItemModifyMode::ModeDelete>(item, qr, ctx); } - void Truncate(const NsContext &ctx) { handleInvalidation(NamespaceImpl::Truncate)(ctx); } + void Delete(const Query &query, QueryResults &result, const RdxContext &ctx) { + nsFuncWrapper<&NamespaceImpl::doDelete, QueryType::QueryDelete>(query, result, ctx); + } + void Truncate(const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Truncate)(ctx); } void Select(QueryResults &result, SelectCtx ¶ms, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Select)(result, params, ctx); } @@ -84,7 +94,7 @@ class Namespace { Item NewItem(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::NewItem)(ctx); } void ToPool(ItemImpl *item) { handleInvalidation(NamespaceImpl::ToPool)(item); } std::string GetMeta(const std::string &key, const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetMeta)(key, ctx); } - void PutMeta(const std::string &key, std::string_view data, const NsContext &ctx) { + void PutMeta(const std::string &key, std::string_view data, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::PutMeta)(key, data, ctx); } int64_t GetSerial(const std::string &field) { return handleInvalidation(NamespaceImpl::GetSerial)(field); } @@ -126,10 +136,11 @@ class Namespace { copyPolicyMultiplier_.store(configData.copyPolicyMultiplier, std::memory_order_relaxed); txSizeToAlwaysCopy_.store(configData.txSizeToAlwaysCopy, std::memory_order_relaxed); longTxLoggingParams_.store(configProvider.GetTxLoggingParams(), std::memory_order_relaxed); + longUpdDelLoggingParams_.store(configProvider.GetUpdDelLoggingParams(), std::memory_order_relaxed); handleInvalidation(NamespaceImpl::OnConfigUpdated)(configProvider, ctx); } StorageOpts GetStorageOpts(const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::GetStorageOpts)(ctx); } - void Refill(std::vector &items, const NsContext &ctx) { handleInvalidation(NamespaceImpl::Refill)(items, ctx); } + void Refill(std::vector &items, const RdxContext &ctx) { handleInvalidation(NamespaceImpl::Refill)(items, ctx); } void DumpIndex(std::ostream &os, std::string_view index, const RdxContext &ctx) { return handleInvalidation(NamespaceImpl::DumpIndex)(os, index, ctx); @@ -168,31 +179,48 @@ class Namespace { } } - template + template void nsFuncWrapper(Item &item, QueryResults &qr, NsContext ctx) const { - nsFuncWrapper(item, qr, ctx); + nsFuncWrapper(item, qr, ctx); } - template + + template void nsFuncWrapper(const Query &query, QueryResults &qr, NsContext ctx) const { - nsFuncWrapper(query, qr, ctx); + nsFuncWrapper(query, qr, + ctx); } - template + + template , ItemModifyMode, QueryType> enumVal> void nsFuncWrapper(T &v, QueryResults &qr, NsContext ctx) const { - assertrx(!ctx.noLock); - ctx.NoLock(); while (true) { - std::shared_ptr ns; + NamespaceImpl::Ptr ns; bool added = false; try { ns = atomicLoadMainNs(); - const auto locker{ns->wLock(ctx.rdxContext)}; - qr.AddNamespace(ns, ctx); - added = true; + PerfStatCalculatorMT calc(ns->updatePerfCounter_, ns->enablePerfCounters_); + CounterGuardAIR32 cg(ns->cancelCommitCnt_); if constexpr (std::is_same_v) { - (*ns.*fn)(v, ctx); + auto locker{ns->wLock(ctx.rdxContext)}; + calc.LockHit(); + cg.Reset(); + ns->checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); + qr.AddNamespace(ns, true); + added = true; + (*ns.*fn)(v, enumVal, ctx); qr.AddItem(v, true, false); - } else { + ns->tryForceFlush(std::move(locker)); + } else if constexpr (std::is_same_v) { + auto params = longUpdDelLoggingParams_.load(std::memory_order_relaxed); + const bool isEnabled = params.thresholdUs >= 0 && !isSystemNamespaceNameFast(v._namespace); + auto statCalculator = QueryStatCalculator(long_actions::MakeLogger(v, std::move(params)), isEnabled); + auto locker = statCalculator.CreateLock(*ns, &NamespaceImpl::wLock, ctx.rdxContext); + calc.LockHit(); + cg.Reset(); + ns->checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); + qr.AddNamespace(ns, true); + added = true; (*ns.*fn)(v, qr, ctx); + statCalculator.LogFlushDuration(*ns, &NamespaceImpl::tryForceFlush, std::move(locker)); } return; } catch (const Error &e) { @@ -217,7 +245,7 @@ class Namespace { ns_.reset(ns); } - std::shared_ptr ns_; + NamespaceImpl::Ptr ns_; std::unique_ptr nsCopy_; std::atomic hasCopy_ = {false}; using Mutex = MarkedMutex; @@ -230,6 +258,8 @@ class Namespace { PerfStatCounterMT commitStatsCounter_; PerfStatCounterMT copyStatsCounter_; std::atomic longTxLoggingParams_; + std::atomic longUpdDelLoggingParams_; + BackgroundNamespaceDeleter &bgDeleter_; }; #undef handleInvalidation diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index 1aa11048f..7848af01e 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -60,7 +60,8 @@ void NamespaceImpl::IndexesStorage::MoveBase(IndexesStorage&& src) { Base::opera // private implementation and NOT THREADSAFE of copy CTOR NamespaceImpl::NamespaceImpl(const NamespaceImpl& src, AsyncStorage::FullLockT& storageLock) - : indexes_{*this}, + : intrusive_atomic_rc_base(), + indexes_{*this}, indexesNames_{src.indexesNames_}, indexesToComposites_{src.indexesToComposites_}, items_{src.items_}, @@ -103,7 +104,8 @@ NamespaceImpl::NamespaceImpl(const NamespaceImpl& src, AsyncStorage::FullLockT& } NamespaceImpl::NamespaceImpl(const std::string& name, UpdatesObservers& observers) - : indexes_(*this), + : intrusive_atomic_rc_base(), + indexes_(*this), name_(name), payloadType_(name), tagsMatcher_(payloadType_), @@ -582,28 +584,7 @@ void NamespaceImpl::addToWAL(std::string_view json, WALRecType type, const RdxCo } void NamespaceImpl::AddIndex(const IndexDef& indexDef, const RdxContext& ctx) { - if (!validateIndexName(indexDef.name_, indexDef.Type())) { - throw Error(errParams, - "Cannot add index '%s' in namespace '%s'. Index name contains invalid characters. Only alphas, digits, '+' (for " - "composite indexes only), '.', '_' " - "and '-' are allowed", - indexDef.name_, name_); - } else if (indexDef.opts_.IsPK()) { - if (indexDef.opts_.IsArray()) { - throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't be array", indexDef.name_, GetName(ctx)); - } else if (indexDef.opts_.IsSparse()) { - throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't be sparse", indexDef.name_, GetName(ctx)); - } else if (isStore(indexDef.Type())) { - throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't have '-' type", indexDef.name_, GetName(ctx)); - } else if (IsFullText(indexDef.Type())) { - throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't be fulltext index", indexDef.name_, - GetName(ctx)); - } - } else if (indexDef.Type() == IndexUuidHash) { - if (indexDef.opts_.IsSparse()) { - throw Error(errParams, "Cannot add index '%s' in namespace '%s'. UUID field can't be sparse", indexDef.name_, GetName(ctx)); - } - } + verifyAddIndex(indexDef, [this, &ctx]() { return GetName(ctx); }); auto wlck = wLock(ctx); @@ -778,6 +759,48 @@ void NamespaceImpl::verifyCompositeIndex(const IndexDef& indexDef) const { if (idx->IsUuid() && type != IndexCompositeHash) { throw Error{errParams, "Only hash index allowed on UUID field"}; } + if (IsComposite(idx->Type())) { + throw Error(errParams, "Cannot create composite index '%s' over the other composite '%s'", indexDef.name_, idx->Name()); + } + } +} + +template +void NamespaceImpl::verifyAddIndex(const IndexDef& indexDef, GetNameF&& getNameF) const { + const auto idxType = indexDef.Type(); + if (!validateIndexName(indexDef.name_, idxType)) { + throw Error(errParams, + "Cannot add index '%s' in namespace '%s'. Index name contains invalid characters. Only alphas, digits, '+' (for " + "composite indexes only), '.', '_' and '-' are allowed", + indexDef.name_, getNameF()); + } + if (indexDef.opts_.IsPK()) { + if (indexDef.opts_.IsArray()) { + throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't be array", indexDef.name_, getNameF()); + } else if (indexDef.opts_.IsSparse()) { + throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't be sparse", indexDef.name_, getNameF()); + } else if (isStore(idxType)) { + throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't have '-' type", indexDef.name_, getNameF()); + } else if (IsFullText(idxType)) { + throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't be fulltext index", indexDef.name_, getNameF()); + } + } + if ((idxType == IndexUuidHash || idxType == IndexUuidStore) && indexDef.opts_.IsSparse()) { + throw Error(errParams, "Cannot add index '%s' in namespace '%s'. UUID field can't be sparse", indexDef.name_, getNameF()); + } + if (indexDef.jsonPaths_.size() > 1 && !IsComposite(idxType) && !indexDef.opts_.IsArray()) { + throw Error(errParams, + "Cannot add index '%s' in namespace '%s'. Scalar (non-array and non-composite) index can not have multiple JSON-paths. " + "Use array index instead", + indexDef.name_, getNameF()); + } + if (indexDef.jsonPaths_.empty()) { + throw Error(errParams, "Cannot add index '%s' in namespace '%s'. JSON paths array can not be empty", indexDef.name_, getNameF()); + } + for (const auto& jp : indexDef.jsonPaths_) { + if (jp.empty()) { + throw Error(errParams, "Cannot add index '%s' in namespace '%s'. JSON path can not be empty", indexDef.name_, getNameF()); + } } } @@ -803,6 +826,20 @@ void NamespaceImpl::verifyUpdateIndex(const IndexDef& indexDef) const { if (indexDef.opts_.IsPK() && isStore(indexDef.Type())) { throw Error(errParams, "Cannot add index '%s' in namespace '%s'. PK field can't have '-' type", indexDef.name_, name_); } + if (indexDef.jsonPaths_.size() > 1 && !IsComposite(indexDef.Type()) && !indexDef.opts_.IsArray()) { + throw Error( + errParams, + "Cannot update index '%s' in namespace '%s'. Scalar (non-array and non-composite) index can not have multiple JSON-paths", + indexDef.name_, name_); + } + if (indexDef.jsonPaths_.empty()) { + throw Error(errParams, "Cannot update index '%s' in namespace '%s'. JSON paths array can not be empty", indexDef.name_, name_); + } + for (const auto& jp : indexDef.jsonPaths_) { + if (jp.empty()) { + throw Error(errParams, "Cannot update index '%s' in namespace '%s'. JSON path can not be empty", indexDef.name_, name_); + } + } if (IsComposite(indexDef.Type())) { verifyUpdateCompositeIndex(indexDef); @@ -811,14 +848,11 @@ void NamespaceImpl::verifyUpdateIndex(const IndexDef& indexDef) const { const auto newIndex = std::unique_ptr(Index::New(indexDef, PayloadType(), FieldsSet())); if (indexDef.opts_.IsSparse()) { - const auto newSparseIndex = std::unique_ptr(Index::New(indexDef, payloadType_, {})); if (indexDef.jsonPaths_.size() != 1) { - throw Error(errParams, "Sparse index must have excatly 1 JSON-path, but %d paths found for '%s'", indexDef.jsonPaths_.size(), + throw Error(errParams, "Sparse index must have exactly 1 JSON-path, but %d paths found for '%s'", indexDef.jsonPaths_.size(), indexDef.name_); } - if (indexDef.jsonPaths_[0].empty()) { - throw Error(errParams, "JSON path for sparse index can not be empty ('%s')", indexDef.name_); - } + const auto newSparseIndex = std::unique_ptr(Index::New(indexDef, payloadType_, {})); } else { FieldsSet changedFields{idxNameIt->second}; PayloadType newPlType = payloadType_; @@ -983,12 +1017,9 @@ void NamespaceImpl::addIndex(const IndexDef& indexDef) { RollBack_addIndex rollbacker{*this}; if (indexDef.opts_.IsSparse()) { if (jsonPaths.size() != 1) { - throw Error(errParams, "Sparse index must have excatly 1 JSON-path, but %d paths found for '%s':'%s'", jsonPaths.size(), name_, + throw Error(errParams, "Sparse index must have exactly 1 JSON-path, but %d paths found for '%s':'%s'", jsonPaths.size(), name_, indexDef.name_); } - if (jsonPaths[0].empty()) { - throw Error(errParams, "JSON path for sparse index('%s':'%s') can not be empty)", name_, indexDef.name_); - } FieldsSet fields; fields.push_back(jsonPaths[0]); TagsPath tagsPath = tagsMatcher_.path2tag(jsonPaths[0], true); @@ -1191,23 +1222,11 @@ bool NamespaceImpl::getSparseIndexByJsonPath(std::string_view jsonPath, int& ind return false; } -void NamespaceImpl::Insert(Item& item, const NsContext& ctx) { modifyItem(item, ctx, ModeInsert); } - -void NamespaceImpl::Update(Item& item, const NsContext& ctx) { modifyItem(item, ctx, ModeUpdate); } - -void NamespaceImpl::Update(const Query& query, QueryResults& result, const NsContext& ctx) { - PerfStatCalculatorMT calc(updatePerfCounter_, enablePerfCounters_); - Locker::WLockT wlck; - - if (!ctx.noLock) { - CounterGuardAIR32 cg(cancelCommitCnt_); - wlck = wLock(ctx.rdxContext); - cg.Reset(); - } - calc.LockHit(); +void NamespaceImpl::Insert(Item& item, const RdxContext& ctx) { ModifyItem(item, ModeInsert, ctx); } - checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); // throw exception if false +void NamespaceImpl::Update(Item& item, const RdxContext& ctx) { ModifyItem(item, ModeUpdate, ctx); } +void NamespaceImpl::doUpdate(const Query& query, QueryResults& result, const NsContext& ctx) { NsSelecter selecter(this); SelectCtx selCtx(query, nullptr); SelectFunctionsHolder func; @@ -1252,7 +1271,7 @@ void NamespaceImpl::Update(const Query& query, QueryResults& result, const NsCon Payload pl(payloadType_, pv); uint64_t oldPlHash = pl.GetHash(); size_t oldItemCapacity = pv.GetCapacity(); - itemModifier.Modify(item.Id(), ctx); + itemModifier.Modify(item.Id()); replicateItem(item.Id(), ctx, statementReplication, oldPlHash, oldItemCapacity); item.Value() = items_[item.Id()]; } @@ -1273,12 +1292,10 @@ void NamespaceImpl::Update(const Query& query, QueryResults& result, const NsCon if (!ctx.rdxContext.fromReplication_) setReplLSNs(LSNPair(lsn_t(), lsn)); } - if (rx_unlikely(query.debugLevel >= LogInfo)) { + if rx_unlikely (query.debugLevel >= LogInfo) { logPrintf(LogInfo, "Updated %d items in %d µs", result.Count(), duration_cast(high_resolution_clock::now() - tmStart).count()); } - - tryForceFlush(std::move(wlck)); } void NamespaceImpl::replicateItem(IdType itemId, const NsContext& ctx, bool statementReplication, uint64_t oldPlHash, @@ -1315,41 +1332,20 @@ void NamespaceImpl::replicateItem(IdType itemId, const NsContext& ctx, bool stat } } -void NamespaceImpl::Upsert(Item& item, const NsContext& ctx) { modifyItem(item, ctx, ModeUpsert); } +void NamespaceImpl::Upsert(Item& item, const RdxContext& ctx) { ModifyItem(item, ModeUpsert, ctx); } -void NamespaceImpl::Delete(Item& item, const NsContext& ctx) { - ItemImpl* ritem = item.impl_; +void NamespaceImpl::Delete(Item& item, const RdxContext& ctx) { ModifyItem(item, ModeDelete, ctx); } +void NamespaceImpl::ModifyItem(Item& item, ItemModifyMode mode, const RdxContext& ctx) { PerfStatCalculatorMT calc(updatePerfCounter_, enablePerfCounters_); - Locker::WLockT wlck; - - if (!ctx.noLock) { - CounterGuardAIR32 cg(cancelCommitCnt_); - wlck = wLock(ctx.rdxContext); - cg.Reset(); - } + CounterGuardAIR32 cg(cancelCommitCnt_); + auto wlck = wLock(ctx); + cg.Reset(); calc.LockHit(); - checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); - - updateTagsMatcherFromItem(ritem); - - auto itItem = findByPK(ritem, ctx.inTransaction, ctx.rdxContext); - IdType id = itItem.first; - - if (!itItem.second) { - return; - } - - item.setID(id); - - WALRecord wrec{WalItemModify, ritem->GetCJSON(), ritem->tagsMatcher().version(), ModeDelete, ctx.inTransaction}; - ritem->RealValue() = items_[id]; - doDelete(id); - - lsn_t itemLsn(item.GetLSN()); - processWalRecord(wrec, ctx.rdxContext, itemLsn, &item); + checkApplySlaveUpdate(ctx.fromReplication_); + modifyItem(item, mode, ctx); tryForceFlush(std::move(wlck)); } @@ -1417,19 +1413,7 @@ void NamespaceImpl::doDelete(IdType id) { markUpdated(true); } -void NamespaceImpl::Delete(const Query& q, QueryResults& result, const NsContext& ctx) { - PerfStatCalculatorMT calc(updatePerfCounter_, enablePerfCounters_); - - Locker::WLockT wlck; - if (!ctx.noLock) { - CounterGuardAIR32 cg(cancelCommitCnt_); - wlck = wLock(ctx.rdxContext); - cg.Reset(); - } - calc.LockHit(); - - checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); // throw exception if false - +void NamespaceImpl::doDelete(const Query& q, QueryResults& result, const NsContext& ctx) { NsSelecter selecter(this); SelectCtx selCtx(q, nullptr); selCtx.contextCollectingMode = true; @@ -1463,11 +1447,10 @@ void NamespaceImpl::Delete(const Query& q, QueryResults& result, const NsContext processWalRecord(wrec, ctx.rdxContext); } } - if (rx_unlikely(q.debugLevel >= LogInfo)) { + if rx_unlikely (q.debugLevel >= LogInfo) { logPrintf(LogInfo, "Deleted %d items in %d µs", result.Count(), duration_cast(high_resolution_clock::now() - tmStart).count()); } - tryForceFlush(std::move(wlck)); } void NamespaceImpl::removeIndex(std::unique_ptr& idx) { @@ -1476,19 +1459,20 @@ void NamespaceImpl::removeIndex(std::unique_ptr& idx) { } } -void NamespaceImpl::Truncate(const NsContext& ctx) { +void NamespaceImpl::Truncate(const RdxContext& ctx) { PerfStatCalculatorMT calc(updatePerfCounter_, enablePerfCounters_); - Locker::WLockT wlck; - if (!ctx.noLock) { - CounterGuardAIR32 cg(cancelCommitCnt_); - wlck = wLock(ctx.rdxContext); - cg.Reset(); - } + CounterGuardAIR32 cg(cancelCommitCnt_); + auto wlck = wLock(ctx); + cg.Reset(); calc.LockHit(); - checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); // throw exception if false + checkApplySlaveUpdate(ctx.fromReplication_); + doTruncate(ctx); + tryForceFlush(std::move(wlck)); +} +void NamespaceImpl::doTruncate(const NsContext& ctx) { if (storage_.IsValid()) { for (PayloadValue& pv : items_) { if (pv.IsFree()) continue; @@ -1520,16 +1504,17 @@ void NamespaceImpl::Truncate(const NsContext& ctx) { if (!repl_.temporary) observers_->OnWALUpdate(LSNPair(lsn, ctx.rdxContext.fromReplication_ ? ctx.rdxContext.LSNs_.originLSN_ : lsn), name_, wrec); if (!ctx.rdxContext.fromReplication_) setReplLSNs(LSNPair(lsn_t(), lsn)); - tryForceFlush(std::move(wlck)); } -void NamespaceImpl::Refill(std::vector& items, const NsContext& ctx) { - auto wlck = wLock(ctx.rdxContext); - auto intCtx = ctx; - intCtx.NoLock(); - Truncate(intCtx); +void NamespaceImpl::Refill(std::vector& items, const RdxContext& ctx) { + auto wlck = wLock(ctx); + + assertrx_throw(isSystem()); + + checkApplySlaveUpdate(ctx.fromReplication_); + doTruncate(ctx); for (Item& i : items) { - Upsert(i, intCtx); + doModifyItem(i, ModeUpsert, ctx); } tryForceFlush(std::move(wlck)); } @@ -1596,7 +1581,7 @@ void NamespaceImpl::CommitTransaction(Transaction& tx, QueryResults& result, NsC QueryStatCalculator& queryStatCalculator) { logPrintf(LogTrace, "[repl:%s]:%d CommitTransaction start", name_, serverId_); Locker::WLockT wlck; - if (!ctx.noLock) { + if (!ctx.isCopiedNsRequest) { PerfStatCalculatorMT calc(updatePerfCounter_, enablePerfCounters_); CounterGuardAIR32 cg(cancelCommitCnt_); wlck = queryStatCalculator.CreateLock(*this, &NamespaceImpl::wLock, ctx.rdxContext); @@ -1604,7 +1589,9 @@ void NamespaceImpl::CommitTransaction(Transaction& tx, QueryResults& result, NsC calc.LockHit(); } - ctx.NoLock().InTransaction(); + checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); + + ctx.InTransaction(); WALRecord initWrec(WalInitTransaction, 0, true); lsn_t lsn(wal_.Add(initWrec), serverId_); @@ -1620,20 +1607,16 @@ void NamespaceImpl::CommitTransaction(Transaction& tx, QueryResults& result, NsC for (auto& step : tx.GetSteps()) { if (step.query_) { QueryResults qr; - qr.AddNamespace(std::shared_ptr{this, [](NamespaceImpl*) {}}, ctx); + qr.AddNamespace(this, true); if (step.query_->type_ == QueryDelete) { - Delete(*step.query_, qr, ctx); + doDelete(*step.query_, qr, ctx); } else { - Update(*step.query_, qr, ctx); + doUpdate(*step.query_, qr, ctx); } } else { const auto modifyMode = step.modifyMode_; Item item = tx.GetItem(std::move(step)); - if (modifyMode == ModeDelete) { - Delete(item, ctx); - } else { - modifyItem(item, ctx, modifyMode); - } + modifyItem(item, modifyMode, ctx); result.AddItem(item); } } @@ -1806,20 +1789,40 @@ void NamespaceImpl::updateTagsMatcherFromItem(ItemImpl* ritem) { } } -void NamespaceImpl::modifyItem(Item& item, const NsContext& ctx, int mode) { - // Item to doUpsert - ItemImpl* itemImpl = item.impl_; - Locker::WLockT wlck; - PerfStatCalculatorMT calc(updatePerfCounter_, enablePerfCounters_); +void NamespaceImpl::modifyItem(Item& item, ItemModifyMode mode, const NsContext& ctx) { + if (mode == ModeDelete) { + deleteItem(item, ctx); + } else { + doModifyItem(item, mode, ctx); + } +} - if (!ctx.noLock) { - CounterGuardAIR32 cg(cancelCommitCnt_); - wlck = wLock(ctx.rdxContext); - cg.Reset(); +void NamespaceImpl::deleteItem(Item& item, const NsContext& ctx) { + ItemImpl* ritem = item.impl_; + + updateTagsMatcherFromItem(ritem); + + auto itItem = findByPK(ritem, ctx.inTransaction, ctx.rdxContext); + IdType id = itItem.first; + + if (!itItem.second) { + return; } - calc.LockHit(); - checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); + item.setID(id); + + WALRecord wrec{WalItemModify, ritem->GetCJSON(), ritem->tagsMatcher().version(), ModeDelete, ctx.inTransaction}; + ritem->RealValue() = items_[id]; + doDelete(id); + + lsn_t itemLsn(item.GetLSN()); + processWalRecord(wrec, ctx.rdxContext, itemLsn, &item); +} + +void NamespaceImpl::doModifyItem(Item& item, ItemModifyMode mode, const NsContext& ctx) { + // Item to doUpsert + assertrx(mode != ModeDelete); + ItemImpl* itemImpl = item.impl_; setFieldsBasedOnPrecepts(itemImpl); updateTagsMatcherFromItem(itemImpl); @@ -1861,8 +1864,6 @@ void NamespaceImpl::modifyItem(Item& item, const NsContext& ctx, int mode) { if (!ctx.rdxContext.fromReplication_) setReplLSNs(LSNPair(lsn_t(), lsn)); markUpdated(!exists); - - tryForceFlush(std::move(wlck)); } // find id by PK. NOT THREAD SAFE! @@ -1903,7 +1904,7 @@ void NamespaceImpl::optimizeIndexes(const NsContext& ctx) { auto lastUpdateTime = lastUpdateTime_.load(std::memory_order_acquire); Locker::RLockT rlck; - if (!ctx.noLock) { + if (!ctx.isCopiedNsRequest) { rlck = rLock(ctx.rdxContext); } @@ -2256,6 +2257,7 @@ bool NamespaceImpl::loadIndexesFromStorage() { Error err = indexDef.FromJSON(giftStr(indexData)); if (err.ok()) { try { + verifyAddIndex(indexDef, [this]() { return this->name_; }); addIndex(indexDef); } catch (const Error& e) { err = e; @@ -2486,15 +2488,14 @@ void NamespaceImpl::removeExpiredItems(RdxActivityContext* ctx) { return; } lastExpirationCheckTs_ = now; - const NsContext nsCtx{rdxCtx, true}; for (const std::unique_ptr& index : indexes_) { if ((index->Type() != IndexTtl) || (index->Size() == 0)) continue; const int64_t expirationthreshold = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() - index->GetTTLValue(); QueryResults qr; - qr.AddNamespace(std::shared_ptr{this, [](NamespaceImpl*) {}}, nsCtx); - Delete(Query(name_).Where(index->Name(), CondLt, expirationthreshold), qr, NsContext(rdxCtx).NoLock()); + qr.AddNamespace(this, true); + doDelete(Query(name_).Where(index->Name(), CondLt, expirationthreshold), qr, rdxCtx); } tryForceFlush(std::move(wlck)); } @@ -2517,16 +2518,6 @@ void NamespaceImpl::removeExpiredStrings(RdxActivityContext* ctx) { } } -StringsHolderPtr NamespaceImpl::StrHolder(const NsContext& ctx) { - assertrx(ctx.noLock); - Locker::RLockT rlck; - if (!ctx.noLock) { - rlck = rLock(ctx.rdxContext); - } - StringsHolderPtr ret{strHolder_}; - return ret; -} - void NamespaceImpl::BackgroundRoutine(RdxActivityContext* ctx) { const RdxContext rdxCtx(ctx); const NsContext nsCtx(rdxCtx); @@ -2580,11 +2571,12 @@ void NamespaceImpl::writeSysRecToStorage(std::string_view data, std::string_view } } -Item NamespaceImpl::NewItem(const NsContext& ctx) { - Locker::RLockT rlck; - if (!ctx.noLock) { - rlck = rLock(ctx.rdxContext); - } +Item NamespaceImpl::NewItem(const RdxContext& ctx) { + auto rlck = rLock(ctx); + return newItem(); +} + +Item NamespaceImpl::newItem() { auto impl_ = pool_.get(0, payloadType_, tagsMatcher_, pkFields(), schema_); impl_->tagsMatcher() = tagsMatcher_; impl_->tagsMatcher().clearUpdated(); @@ -2618,10 +2610,10 @@ std::string NamespaceImpl::getMeta(const std::string& key) const { } // Put meta data to storage by key -void NamespaceImpl::PutMeta(const std::string& key, std::string_view data, const NsContext& ctx) { - auto wlck = wLock(ctx.rdxContext); - checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); // throw exception if false - putMeta(key, data, ctx.rdxContext); +void NamespaceImpl::PutMeta(const std::string& key, std::string_view data, const RdxContext& ctx) { + auto wlck = wLock(ctx); + checkApplySlaveUpdate(ctx.fromReplication_); + putMeta(key, data, ctx); } // Put meta data to storage by key diff --git a/cpp_src/core/namespace/namespaceimpl.h b/cpp_src/core/namespace/namespaceimpl.h index fbb0ef6ca..46fa4d1e1 100644 --- a/cpp_src/core/namespace/namespaceimpl.h +++ b/cpp_src/core/namespace/namespaceimpl.h @@ -57,27 +57,34 @@ class QueryResults; namespace long_actions { template struct Logger; -} + +template +struct QueryEnum2Type; +} // namespace long_actions + template class> class QueryStatCalculator; +template +using QueryStatCalculatorUpdDel = QueryStatCalculator, long_actions::Logger>; + namespace SortExprFuncs { struct DistanceBetweenJoinedIndexesSameNs; } // namespace SortExprFuncs struct NsContext { - NsContext(const RdxContext &rdxCtx, bool noLock_ = false) noexcept : rdxContext{rdxCtx}, noLock{noLock_} {} - NsContext &NoLock() noexcept { - noLock = true; - return *this; - } + NsContext(const RdxContext &rdxCtx) noexcept : rdxContext{rdxCtx} {} NsContext &InTransaction() noexcept { inTransaction = true; return *this; } + NsContext &CopiedNsRequest() noexcept { + isCopiedNsRequest = true; + return *this; + } const RdxContext &rdxContext; - bool noLock = false; + bool isCopiedNsRequest = false; bool inTransaction = false; }; @@ -85,7 +92,7 @@ namespace composite_substitution_helpers { class CompositeSearcher; } -class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter for this class +class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance.Padding) Padding does not matter for this class class RollBack_insertIndex; class RollBack_addIndex; template @@ -100,7 +107,7 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f IndexesCacheCleaner &operator=(const IndexesCacheCleaner &) = delete; IndexesCacheCleaner &operator=(IndexesCacheCleaner &&) = delete; void Add(SortType s) { - if (rx_unlikely(s >= sorts_.size())) { + if rx_unlikely (s >= sorts_.size()) { throw Error(errLogic, "Index sort type overflow: %d. Limit is %d", s, sorts_.size() - 1); } if (s > 0) { @@ -189,7 +196,7 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f public: enum OptimizationState : int { NotOptimized, OptimizedPartially, OptimizationCompleted }; - typedef shared_ptr Ptr; + using Ptr = intrusive_ptr; using Mutex = MarkedMutex; NamespaceImpl(const std::string &_name, UpdatesObservers &observers); @@ -218,15 +225,13 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f void SetSchema(std::string_view schema, const RdxContext &ctx); std::string GetSchema(int format, const RdxContext &ctx); - void Insert(Item &item, const NsContext &ctx); - void Update(Item &item, const NsContext &ctx); - void Update(const Query &query, QueryResults &result, const NsContext &); - void Upsert(Item &item, const NsContext &); - - void Delete(Item &item, const NsContext &); - void Delete(const Query &query, QueryResults &result, const NsContext &); - void Truncate(const NsContext &); - void Refill(std::vector &, const NsContext &); + void Insert(Item &item, const RdxContext &ctx); + void Update(Item &item, const RdxContext &ctx); + void Upsert(Item &item, const RdxContext &); + void Delete(Item &item, const RdxContext &); + void ModifyItem(Item &item, ItemModifyMode mode, const RdxContext &ctx); + void Truncate(const RdxContext &); + void Refill(std::vector &, const RdxContext &); void Select(QueryResults &result, SelectCtx ¶ms, const RdxContext &); NamespaceDef GetDefinition(const RdxContext &ctx); @@ -243,12 +248,12 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f void CommitTransaction(Transaction &tx, QueryResults &result, NsContext ctx, QueryStatCalculator &queryStatCalculator); - Item NewItem(const NsContext &ctx); + Item NewItem(const RdxContext &ctx); void ToPool(ItemImpl *item); // Get meta data from storage by key std::string GetMeta(const std::string &key, const RdxContext &ctx); // Put meta data to storage by key - void PutMeta(const std::string &key, std::string_view data, const NsContext &); + void PutMeta(const std::string &key, std::string_view data, const RdxContext &); int64_t GetSerial(const std::string &field); int getIndexByName(std::string_view index) const; @@ -322,8 +327,13 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f void initWAL(int64_t minLSN, int64_t maxLSN); void markUpdated(bool forceOptimizeAllIndexes); + void doUpdate(const Query &query, QueryResults &result, const NsContext &); + void doDelete(const Query &query, QueryResults &result, const NsContext &); + void doTruncate(const NsContext &ctx); void doUpsert(ItemImpl *ritem, IdType id, bool doUpdate); - void modifyItem(Item &item, const NsContext &, int mode = ModeUpsert); + void modifyItem(Item &item, ItemModifyMode mode, const NsContext &); + void doModifyItem(Item &item, ItemModifyMode mode, const NsContext &ctx); + void deleteItem(Item &item, const NsContext &ctx); void updateTagsMatcherFromItem(ItemImpl *ritem); template [[nodiscard]] RollBack_updateItems updateItems(const PayloadType &oldPlType, const FieldsSet &changedFields, @@ -337,6 +347,8 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f template void createFieldsSet(const std::string &idxName, IndexType type, const PathsT &paths, FieldsSet &fields); void verifyCompositeIndex(const IndexDef &indexDef) const; + template + void verifyAddIndex(const IndexDef &indexDef, GetNameF &&) const; void verifyUpdateIndex(const IndexDef &indexDef) const; void verifyUpdateCompositeIndex(const IndexDef &indexDef) const; void updateIndex(const IndexDef &indexDef); @@ -346,6 +358,7 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f void replicateItem(IdType itemId, const NsContext &ctx, bool statementReplication, uint64_t oldPlHash, size_t oldItemCapacity); void removeExpiredItems(RdxActivityContext *); void removeExpiredStrings(RdxActivityContext *); + Item newItem(); template [[nodiscard]] RollBack_recreateCompositeIndexes recreateCompositeIndexes(size_t startIdx, size_t endIdx); @@ -410,7 +423,7 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f Locker locker_; std::shared_ptr schema_; - StringsHolderPtr StrHolder(const NsContext &); + StringsHolderPtr strHolder() const noexcept { return strHolder_; } size_t ItemsCount() const noexcept { return items_.size() - free_.size(); } const NamespaceConfigData &Config() const noexcept { return config_; } @@ -470,5 +483,4 @@ class NamespaceImpl { // NOLINT(*performance.Padding) Padding does not matter f mutable std::atomic nsUpdateSortedContextMemory_ = {0}; std::atomic dbDestroyed_{false}; }; - } // namespace reindexer diff --git a/cpp_src/core/nsselecter/explaincalc.cc b/cpp_src/core/nsselecter/explaincalc.cc index 86cd6c7d0..c7d9506de 100644 --- a/cpp_src/core/nsselecter/explaincalc.cc +++ b/cpp_src/core/nsselecter/explaincalc.cc @@ -1,9 +1,8 @@ #include "explaincalc.h" + #include -#include "core/cbinding/reindexer_ctypes.h" + #include "core/cjson/jsonbuilder.h" -#include "core/namespace/namespaceimpl.h" -#include "core/query/sql/sqlencoder.h" #include "nsselecter.h" #include "tools/logger.h" @@ -36,9 +35,9 @@ void ExplainCalc::LogDump(int logLevel) { if (jselectors_) { for (auto &js : *jselectors_) { if (js.Type() == JoinType::LeftJoin || js.Type() == JoinType::Merge) { - logPrintf(LogInfo, "%s %s: called %d", SQLEncoder::JoinTypeName(js.Type()), js.RightNsName(), js.Called()); + logPrintf(LogInfo, "%s %s: called %d", JoinTypeName(js.Type()), js.RightNsName(), js.Called()); } else { - logPrintf(LogInfo, "%s %s: called %d, matched %d", SQLEncoder::JoinTypeName(js.Type()), js.RightNsName(), js.Called(), + logPrintf(LogInfo, "%s %s: called %d, matched %d", JoinTypeName(js.Type()), js.RightNsName(), js.Called(), js.Matched()); } } @@ -70,17 +69,32 @@ constexpr inline const char *opName(OpType op, bool first = true) { case OpNot: return "not "; default: - throw Error(errLogic, "Unexpected op type: %d", int(op)); + throw Error(errLogic, "Unexpected op type %d", int(op)); + } +} + +constexpr std::string_view fieldKind(IteratorFieldKind fk) { + using namespace std::string_view_literals; + switch (fk) { + case IteratorFieldKind::NonIndexed: + return "non-indexed"sv; + case IteratorFieldKind::Indexed: + return "indexed"sv; + case IteratorFieldKind::None: + return ""sv; + default: + throw Error(errLogic, "Unexpected field type %d", int(fk)); } } static std::string addToJSON(JsonBuilder &builder, const JoinedSelector &js, OpType op = OpAnd) { + using namespace std::string_view_literals; auto jsonSel = builder.Object(); std::string name{joinTypeName(js.Type()) + js.RightNsName()}; - jsonSel.Put("field", opName(op) + name); - jsonSel.Put("matched", js.Matched()); - jsonSel.Put("selects_count", js.Called()); - jsonSel.Put("join_select_total", ExplainCalc::To_us(js.PreResult()->selectTime)); + jsonSel.Put("field"sv, opName(op) + name); + jsonSel.Put("matched"sv, js.Matched()); + jsonSel.Put("selects_count"sv, js.Called()); + jsonSel.Put("join_select_total"sv, ExplainCalc::To_us(js.PreResult()->selectTime)); switch (js.Type()) { case JoinType::InnerJoin: case JoinType::OrInnerJoin: @@ -88,25 +102,25 @@ static std::string addToJSON(JsonBuilder &builder, const JoinedSelector &js, OpT assertrx(js.PreResult()); switch (js.PreResult()->dataMode) { case JoinPreResult::ModeValues: - jsonSel.Put("method", "preselected_values"); - jsonSel.Put("keys", js.PreResult()->values.size()); + jsonSel.Put("method"sv, "preselected_values"sv); + jsonSel.Put("keys"sv, js.PreResult()->values.size()); break; case JoinPreResult::ModeIdSet: - jsonSel.Put("method", "preselected_rows"); - jsonSel.Put("keys", js.PreResult()->ids.size()); + jsonSel.Put("method"sv, "preselected_rows"sv); + jsonSel.Put("keys"sv, js.PreResult()->ids.size()); break; case JoinPreResult::ModeIterators: - jsonSel.Put("method", "no_preselect"); - jsonSel.Put("keys", js.PreResult()->iterators.Size()); + jsonSel.Put("method"sv, "no_preselect"sv); + jsonSel.Put("keys"sv, js.PreResult()->iterators.Size()); break; default: break; } if (!js.PreResult()->explainPreSelect.empty()) { - jsonSel.Raw("explain_preselect", js.PreResult()->explainPreSelect); + jsonSel.Raw("explain_preselect"sv, js.PreResult()->explainPreSelect); } if (!js.PreResult()->explainOneSelect.empty()) { - jsonSel.Raw("explain_select", js.PreResult()->explainOneSelect); + jsonSel.Raw("explain_select"sv, js.PreResult()->explainOneSelect); } break; case JoinType::Merge: @@ -115,31 +129,101 @@ static std::string addToJSON(JsonBuilder &builder, const JoinedSelector &js, OpT return name; } +static void addToJSON(JsonBuilder &builder, const ConditionInjection &injCond) { + auto jsonSel = builder.Object(); + using namespace std::string_view_literals; + using namespace std::string_literals; + + jsonSel.Put("condition"sv, injCond.initCond); + jsonSel.Put("total_time_us"sv, ExplainCalc::To_us(injCond.totalTime_)); + jsonSel.Put("success"sv, injCond.succeed); + if (!injCond.succeed) { + if (injCond.reason.empty()) { + if (injCond.orChainPart_) { + jsonSel.Put("reason"sv, "Skipped as Or-chain part."sv); + } else { + jsonSel.Put("reason"sv, "Unknown"sv); + } + } else { + std::string reason{injCond.reason}; + if (injCond.orChainPart_) { + reason += " Or-chain part."; + } + jsonSel.Put("reason"sv, reason); + } + } + + if (!injCond.explain.empty()) { + jsonSel.Raw("explain_select"sv, injCond.explain); + } + if (injCond.aggType != AggType::AggUnknown) { + jsonSel.Put("agg_type"sv, AggTypeToStr(injCond.aggType)); + } + jsonSel.Put("values_count"sv, injCond.valuesCount); + jsonSel.Put("new_condition"sv, injCond.newCond); +} + +static std::string addToJSON(JsonBuilder &builder, const JoinOnInjection &injCond) { + auto jsonSel = builder.Object(); + std::string name{injCond.rightNsName}; + using namespace std::string_view_literals; + + jsonSel.Put("namespace"sv, injCond.rightNsName); + jsonSel.Put("on_condition"sv, injCond.joinCond); + jsonSel.Put("type"sv, injCond.type == JoinOnInjection::ByValue ? "by_value"sv : "select"sv); + jsonSel.Put("total_time_us"sv, ExplainCalc::To_us(injCond.totalTime_)); + jsonSel.Put("success"sv, injCond.succeed); + if (!injCond.reason.empty()) { + jsonSel.Put("reason"sv, injCond.reason); + } + jsonSel.Put("injected_condition"sv, injCond.injectedCond.Slice()); + if (!injCond.conditions.empty()) { + auto jsonCondInjections = jsonSel.Array("conditions"sv); + for (const auto &cond : injCond.conditions) { + addToJSON(jsonCondInjections, cond); + } + } + + return name; +} + std::string ExplainCalc::GetJSON() { + using namespace std::string_view_literals; WrSerializer ser; { JsonBuilder json(ser); if (enabled_) { - json.Put("total_us", To_us(total_)); - json.Put("prepare_us", To_us(prepare_)); - json.Put("indexes_us", To_us(select_)); - json.Put("postprocess_us", To_us(postprocess_)); - json.Put("loop_us", To_us(loop_)); - json.Put("general_sort_us", To_us(sort_)); + json.Put("total_us"sv, To_us(total_)); + json.Put("preselect_us"sv, To_us(preselect_)); + json.Put("prepare_us"sv, To_us(prepare_)); + json.Put("indexes_us"sv, To_us(select_)); + json.Put("postprocess_us"sv, To_us(postprocess_)); + json.Put("loop_us"sv, To_us(loop_)); + json.Put("general_sort_us"sv, To_us(sort_)); } - json.Put("sort_index", sortIndex_); - json.Put("sort_by_uncommitted_index", sortOptimization_); + json.Put("sort_index"sv, sortIndex_); + json.Put("sort_by_uncommitted_index"sv, sortOptimization_); - auto jsonSelArr = json.Array("selectors"); + { + auto jsonSelArr = json.Array("selectors"sv); - if (selectors_) { - selectors_->ExplainJSON(iters_, jsonSelArr, jselectors_); + if (selectors_) { + selectors_->ExplainJSON(iters_, jsonSelArr, jselectors_); + } + + if (jselectors_) { + // adding explain for LeftJoin-s and Merge subqueries + for (const JoinedSelector &js : *jselectors_) { + if (js.Type() == JoinType::InnerJoin || js.Type() == JoinType::OrInnerJoin) continue; + addToJSON(jsonSelArr, js); + } + } } - if (jselectors_) { - for (JoinedSelector &js : *jselectors_) { - if (js.Type() == JoinType::InnerJoin || js.Type() == JoinType::OrInnerJoin) continue; - addToJSON(jsonSelArr, js); + if (onInjections_ && !onInjections_->empty()) { + auto jsonOnInjections = json.Array("on_conditions_injections"sv); + for (const JoinOnInjection &injCond : *onInjections_) { + addToJSON(jsonOnInjections, injCond); } } } @@ -150,6 +234,8 @@ std::string ExplainCalc::GetJSON() { std::string SelectIteratorContainer::explainJSON(const_iterator begin, const_iterator end, int iters, JsonBuilder &builder, const JoinedSelectors *jselectors) { using namespace std::string_literals; + using namespace std::string_view_literals; + std::stringstream name; name << '('; for (const_iterator it = begin; it != end; ++it) { @@ -157,26 +243,29 @@ std::string SelectIteratorContainer::explainJSON(const_iterator begin, const_ite it->InvokeAppropriate( [&](const SelectIteratorsBracket &) { auto jsonSel = builder.Object(); - auto jsonSelArr = jsonSel.Array("selectors"); + auto jsonSelArr = jsonSel.Array("selectors"sv); const std::string brName{explainJSON(it.cbegin(), it.cend(), iters, jsonSelArr, jselectors)}; jsonSelArr.End(); - jsonSel.Put("field", opName(it->operation) + brName); + jsonSel.Put("field"sv, opName(it->operation) + brName); name << opName(it->operation, it == begin) << brName; }, [&](const SelectIterator &siter) { auto jsonSel = builder.Object(); - const bool isScanIterator{std::string_view(siter.name) == "-scan"}; + const bool isScanIterator{std::string_view(siter.name) == "-scan"sv}; if (!isScanIterator) { - jsonSel.Put("keys", siter.size()); - jsonSel.Put("comparators", siter.comparators_.size()); - jsonSel.Put("cost", siter.Cost(iters)); + jsonSel.Put("keys"sv, siter.size()); + jsonSel.Put("comparators"sv, siter.comparators_.size()); + jsonSel.Put("cost"sv, siter.Cost(iters)); } else { - jsonSel.Put("items", siter.GetMaxIterations(iters)); + jsonSel.Put("items"sv, siter.GetMaxIterations(iters)); } - jsonSel.Put("field", opName(it->operation) + siter.name); - jsonSel.Put("matched", siter.GetMatchedCount()); - jsonSel.Put("method", isScanIterator || siter.comparators_.size() ? "scan" : "index"); - jsonSel.Put("type", siter.TypeName()); + jsonSel.Put("field"sv, opName(it->operation) + siter.name); + if (siter.fieldKind != IteratorFieldKind::None) { + jsonSel.Put("field_type"sv, fieldKind(siter.fieldKind)); + } + jsonSel.Put("matched"sv, siter.GetMatchedCount()); + jsonSel.Put("method"sv, isScanIterator || siter.comparators_.size() ? "scan"sv : "index"sv); + jsonSel.Put("type"sv, siter.TypeName()); name << opName(it->operation, it == begin) << siter.name; }, [&](const JoinSelectIterator &jiter) { @@ -186,21 +275,21 @@ std::string SelectIteratorContainer::explainJSON(const_iterator begin, const_ite }, [&](const FieldsComparator &c) { auto jsonSel = builder.Object(); - jsonSel.Put("comparators", 1); - jsonSel.Put("field", opName(it->operation) + c.Name()); - jsonSel.Put("cost", c.Cost(iters)); - jsonSel.Put("method", "scan"); - jsonSel.Put("items", iters); - jsonSel.Put("matched", c.GetMatchedCount()); - jsonSel.Put("type", "TwoFieldsComparison"); + jsonSel.Put("comparators"sv, 1); + jsonSel.Put("field"sv, opName(it->operation) + c.Name()); + jsonSel.Put("cost"sv, c.Cost(iters)); + jsonSel.Put("method"sv, "scan"sv); + jsonSel.Put("items"sv, iters); + jsonSel.Put("matched"sv, c.GetMatchedCount()); + jsonSel.Put("type"sv, "TwoFieldsComparison"sv); name << opName(it->operation, it == begin) << c.Name(); }, [&](const AlwaysFalse &) { auto jsonSkiped = builder.Object(); - jsonSkiped.Put("type", "Skipped"); - jsonSkiped.Put("description", "always "s + (it->operation == OpNot ? "true" : "false")); - name << opName(it->operation == OpNot ? OpAnd : it->operation, it == begin) << "Always" - << (it->operation == OpNot ? "True" : "False"); + jsonSkiped.Put("type"sv, "Skipped"sv); + jsonSkiped.Put("description"sv, "always "s + (it->operation == OpNot ? "true" : "false")); + name << opName(it->operation == OpNot ? OpAnd : it->operation, it == begin) << "Always"sv + << (it->operation == OpNot ? "True"sv : "False"sv); }); } name << ')'; @@ -216,41 +305,36 @@ ExplainCalc::Duration ExplainCalc::lap() noexcept { int ExplainCalc::To_us(const ExplainCalc::Duration &d) noexcept { return duration_cast(d).count(); } -void reindexer::ExplainCalc::StartTiming() noexcept { +void ExplainCalc::StartTiming() noexcept { if (enabled_) lap(); } -void reindexer::ExplainCalc::StopTiming() noexcept { - if (enabled_) total_ = prepare_ + select_ + postprocess_ + loop_; +void ExplainCalc::StopTiming() noexcept { + if (enabled_) total_ = preselect_ + prepare_ + select_ + postprocess_ + loop_; } -void reindexer::ExplainCalc::AddPrepareTime() noexcept { +void ExplainCalc::AddPrepareTime() noexcept { if (enabled_) prepare_ += lap(); } -void reindexer::ExplainCalc::AddSelectTime() noexcept { +void ExplainCalc::AddSelectTime() noexcept { if (enabled_) select_ += lap(); } -void reindexer::ExplainCalc::AddPostprocessTime() noexcept { +void ExplainCalc::AddPostprocessTime() noexcept { if (enabled_) postprocess_ += lap(); } -void reindexer::ExplainCalc::AddLoopTime() noexcept { +void ExplainCalc::AddLoopTime() noexcept { if (enabled_) loop_ += lap(); } -void reindexer::ExplainCalc::StartSort() noexcept { +void ExplainCalc::StartSort() noexcept { if (enabled_) sort_start_point_ = Clock::now(); } -void reindexer::ExplainCalc::StopSort() noexcept { +void ExplainCalc::StopSort() noexcept { if (enabled_) sort_ = Clock::now() - sort_start_point_; } -void reindexer::ExplainCalc::AddIterations(int iters) noexcept { iters_ += iters; } -void reindexer::ExplainCalc::PutSortIndex(std::string_view index) noexcept { sortIndex_ = index; } -void ExplainCalc::PutSelectors(SelectIteratorContainer *qres) noexcept { selectors_ = qres; } -void ExplainCalc::PutJoinedSelectors(JoinedSelectors *jselectors) noexcept { jselectors_ = jselectors; } - } // namespace reindexer diff --git a/cpp_src/core/nsselecter/explaincalc.h b/cpp_src/core/nsselecter/explaincalc.h index 34cccd5f9..837dfafde 100644 --- a/cpp_src/core/nsselecter/explaincalc.h +++ b/cpp_src/core/nsselecter/explaincalc.h @@ -5,13 +5,17 @@ #include #include "core/type_consts.h" -#include "estl/h_vector.h" +#include "tools/serializer.h" namespace reindexer { class SelectIteratorContainer; class JoinedSelector; +struct JoinOnInjection; +struct ConditionInjection; + typedef std::vector JoinedSelectors; +typedef std::vector OnConditionInjections; class ExplainCalc { public: @@ -32,14 +36,16 @@ class ExplainCalc { void AddSelectTime() noexcept; void AddPostprocessTime() noexcept; void AddLoopTime() noexcept; - void AddIterations(int iters) noexcept; + void AddIterations(int iters) noexcept { iters_ += iters; } void StartSort() noexcept; void StopSort() noexcept; void PutCount(int cnt) noexcept { count_ = cnt; } - void PutSortIndex(std::string_view index) noexcept; - void PutSelectors(SelectIteratorContainer *qres) noexcept; - void PutJoinedSelectors(JoinedSelectors *jselectors) noexcept; + void PutSortIndex(std::string_view index) noexcept { sortIndex_ = index; } + void PutSelectors(const SelectIteratorContainer *qres) noexcept { selectors_ = qres; } + void PutJoinedSelectors(const JoinedSelectors *jselectors) noexcept { jselectors_ = jselectors; } + void SetPreselectTime(Duration preselectTime) noexcept { preselect_ = preselectTime; } + void PutOnConditionInjections(const OnConditionInjections *onCondInjections) noexcept { onInjections_ = onCondInjections; } void SetSortOptimization(bool enable) noexcept { sortOptimization_ = enable; } void LogDump(int logLevel); @@ -58,22 +64,56 @@ class ExplainCalc { private: Duration lap() noexcept; - static const char *JoinTypeName(JoinType jtype); time_point last_point_, sort_start_point_; Duration total_, prepare_ = Duration::zero(); + Duration preselect_ = Duration::zero(); Duration select_ = Duration::zero(); Duration postprocess_ = Duration::zero(); Duration loop_ = Duration::zero(); Duration sort_ = Duration::zero(); std::string_view sortIndex_; - SelectIteratorContainer *selectors_ = nullptr; - JoinedSelectors *jselectors_ = nullptr; + const SelectIteratorContainer *selectors_ = nullptr; + const JoinedSelectors *jselectors_ = nullptr; + const OnConditionInjections *onInjections_ = nullptr; ///< Optional + int iters_ = 0; int count_ = 0; bool sortOptimization_ = false; bool enabled_ = false; }; +/** + * @brief Describes the process of a single JOIN-query ON-conditions injection into the Where clause of a main query + */ +struct JoinOnInjection { + std::string_view rightNsName; ///< joinable ns name + std::string joinCond; ///< original ON-conditions clause. SQL-like string + ExplainCalc::Duration totalTime_ = + ExplainCalc::Duration::zero(); ///< total amount of time spent on checking and substituting all conditions + bool succeed = false; ///< result of injection attempt + std::string_view reason; ///< optional{succeed==false}. Explains condition injection failure + enum { ByValue, Select } type = ByValue; ///< byValue or Select + WrSerializer injectedCond; ///< injected condition. SQL-like string + std::vector conditions; ///< individual conditions processing results +}; + +/** + * @brief Describes an injection attempt of a single condition from the ON-clause of a JOIN-query + */ +struct ConditionInjection { + std::string initCond; ///< single condition from Join ON section. SQL-like string + ExplainCalc::Duration totalTime_ = + ExplainCalc::Duration::zero(); ///< total time elapsed from injection attempt start till the end of substitution or rejection + std::string explain; ///< optoinal{JoinOnInjection.type == Select}. Explain raw string from Select subquery. + AggType aggType = AggType::AggUnknown; ///< aggregation type used in subquery + bool succeed = false; ///< result of injection attempt + std::string_view reason; ///< optional{succeed==false}. Explains condition injection failure + bool orChainPart_ = false; ///< additional failure reason flag. Used in case if condition field was filled before and + ///< also it does not fit because it is an OR chain part + std::string newCond; ///< substituted condition in QueryEntry. SQL-like string + size_t valuesCount = 0; ///< resulting size of query values set +}; + } // namespace reindexer diff --git a/cpp_src/core/nsselecter/joinedselector.cc b/cpp_src/core/nsselecter/joinedselector.cc index c276e25f6..84fbe4b8b 100644 --- a/cpp_src/core/nsselecter/joinedselector.cc +++ b/cpp_src/core/nsselecter/joinedselector.cc @@ -1,4 +1,5 @@ #include "joinedselector.h" + #include "core/namespace/namespaceimpl.h" #include "core/queryresults/joinresults.h" #include "nsselecter.h" @@ -229,7 +230,8 @@ void JoinedSelector::AppendSelectIteratorOfJoinIndexData(SelectIteratorContainer bool was = false; for (SelectKeyResult &res : leftIndex->SelectKey(values, CondSet, sortId, opts, ctx, rdxCtx)) { if (!res.comparators_.empty()) continue; - SelectIterator selIter{res, false, joinEntry.index_, false}; + SelectIterator selIter{res, false, joinEntry.index_, + (joinEntry.idxNo < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed), false}; selIter.Bind(leftNs_->payloadType_, joinEntry.idxNo); const int curIterations = selIter.GetMaxIterations(); if (curIterations && curIterations < *maxIterations) *maxIterations = curIterations; diff --git a/cpp_src/core/nsselecter/joinedselector.h b/cpp_src/core/nsselecter/joinedselector.h index be9f66e0a..0fb63758d 100644 --- a/cpp_src/core/nsselecter/joinedselector.h +++ b/cpp_src/core/nsselecter/joinedselector.h @@ -1,5 +1,6 @@ #pragma once #include "core/joincache.h" +#include "core/namespace/namespaceimpl.h" #include "explaincalc.h" #include "selectiteratorcontainer.h" @@ -69,10 +70,9 @@ class JoinedSelector { friend QueryPreprocessor; public: - JoinedSelector(JoinType joinType, std::shared_ptr leftNs, std::shared_ptr rightNs, JoinCacheRes &&joinRes, - Query &&itemQuery, QueryResults &result, const JoinedQuery &joinQuery, JoinPreResult::Ptr preResult, - uint32_t joinedFieldIdx, SelectFunctionsHolder &selectFunctions, uint32_t joinedSelectorsCount, bool inTransaction, - const RdxContext &rdxCtx) + JoinedSelector(JoinType joinType, NamespaceImpl::Ptr leftNs, NamespaceImpl::Ptr rightNs, JoinCacheRes &&joinRes, Query &&itemQuery, + QueryResults &result, const JoinedQuery &joinQuery, JoinPreResult::Ptr preResult, uint32_t joinedFieldIdx, + SelectFunctionsHolder &selectFunctions, uint32_t joinedSelectorsCount, bool inTransaction, const RdxContext &rdxCtx) : joinType_(joinType), called_(0), matched_(0), @@ -106,7 +106,7 @@ class JoinedSelector { const RdxContext &); static constexpr int MaxIterationsForPreResultStoreValuesOptimization() noexcept { return 200; } JoinPreResult::CPtr PreResult() const noexcept { return preResult_; } - const std::shared_ptr &RightNs() const noexcept { return rightNs_; } + const NamespaceImpl::Ptr &RightNs() const noexcept { return rightNs_; } private: template @@ -118,8 +118,8 @@ class JoinedSelector { JoinType joinType_; int called_, matched_; - std::shared_ptr leftNs_; - std::shared_ptr rightNs_; + NamespaceImpl::Ptr leftNs_; + NamespaceImpl::Ptr rightNs_; JoinCacheRes joinRes_; Query itemQuery_; QueryResults &result_; diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index bdb1cf571..0e7bf7650 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -1,4 +1,6 @@ #include "nsselecter.h" + +#include "core/cjson/jsonbuilder.h" #include "core/namespace/namespaceimpl.h" #include "core/queryresults/joinresults.h" #include "crashqueryreporter.h" @@ -30,27 +32,31 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte explain = ExplainCalc(ctx.query.explain_ || ctx.query.debugLevel >= LogInfo); ActiveQueryScope queryScope(ctx, ns_->optimizationState_, explain, ns_->locker_.IsReadOnly(), ns_->strHolder_.get()); + explain.SetPreselectTime(ctx.preResultTimeTotal); explain.StartTiming(); - auto containSomeAggCount = [&ctx](const AggType &type) { - auto it = std::find_if(ctx.query.aggregations_.begin(), ctx.query.aggregations_.end(), - [&type](const AggregateEntry &agg) { return agg.Type() == type; }); - return it != ctx.query.aggregations_.end(); + const auto &aggregationQueryRef = ctx.isMergeQuerySubQuery() ? *ctx.parentQuery : ctx.query; + + auto containSomeAggCount = [&aggregationQueryRef](AggType type) noexcept { + auto it = std::find_if(aggregationQueryRef.aggregations_.begin(), aggregationQueryRef.aggregations_.end(), + [type](const AggregateEntry &agg) { return agg.Type() == type; }); + return it != aggregationQueryRef.aggregations_.end(); }; bool needPutCachedTotal = false; + const auto initTotalCount = result.totalCount; bool containAggCount = containSomeAggCount(AggCount); bool containAggCountCached = containAggCount ? false : containSomeAggCount(AggCountCached); - bool needCalcTotal = ctx.query.calcTotal == ModeAccurateTotal || containAggCount; + bool needCalcTotal = aggregationQueryRef.calcTotal == ModeAccurateTotal || containAggCount; QueryCacheKey ckey; - if (ctx.query.calcTotal == ModeCachedTotal || containAggCountCached) { + if (aggregationQueryRef.calcTotal == ModeCachedTotal || containAggCountCached) { ckey = QueryCacheKey{ctx.query}; auto cached = ns_->queryTotalCountCache_->Get(ckey); if (cached.valid && cached.val.total_count >= 0) { - result.totalCount = cached.val.total_count; + result.totalCount += cached.val.total_count; logPrintf(LogTrace, "[%s] using value from cache: %d", ns_->name_, result.totalCount); } else { needPutCachedTotal = cached.valid; @@ -59,14 +65,17 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte } } + OnConditionInjections explainInjectedOnConditions; QueryPreprocessor qPreproc((ctx.preResult && ctx.preResult->executionMode == JoinPreResult::ModeExecute) ? const_cast(&ctx.query.entries)->MakeLazyCopy() : QueryEntries{ctx.query.entries}, ns_, ctx); if (ctx.joinedSelectors) { - qPreproc.InjectConditionsFromJoins(*ctx.joinedSelectors, rdxCtx); + qPreproc.InjectConditionsFromJoins(*ctx.joinedSelectors, explainInjectedOnConditions, rdxCtx); + explain.PutOnConditionInjections(&explainInjectedOnConditions); } - auto aggregators = getAggregators(ctx.query); + auto aggregators = getAggregators(aggregationQueryRef.aggregations_, aggregationQueryRef.strictMode); + qPreproc.AddDistinctEntries(aggregators); const bool aggregationsOnly = aggregators.size() > 1 || (aggregators.size() == 1 && aggregators[0].Type() != AggDistinct); qPreproc.InitIndexNumbers(); @@ -170,7 +179,8 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte SelectKeyResult res; res.emplace_back(std::move(ctx.preResult->ids)); static const std::string pr = "-preresult"; - qres.Append(OpAnd, SelectIterator(std::move(res), false, pr)); + // Iterator Field Kind: Preselect IdSet -> None + qres.Append(OpAnd, SelectIterator(std::move(res), false, pr, IteratorFieldKind::None)); } break; case JoinPreResult::ModeIterators: qres.LazyAppend(ctx.preResult->iterators.begin(), ctx.preResult->iterators.end()); @@ -202,7 +212,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte !ctx.sortingContext.sortIndex()) // 2. We have sorted query, by unordered index || ctx.preResult->btreeIndexOptimizationEnabled) { // 3. We have btree-index that is not committed yet ctx.preResult->iterators.Append(qres.cbegin(), qres.cend()); - if (rx_unlikely(ctx.query.debugLevel >= LogInfo)) { + if rx_unlikely (ctx.query.debugLevel >= LogInfo) { logPrintf(LogInfo, "Built preResult (expected %d iterations) with %d iterators, q='%s'", maxIterations, qres.Size(), ctx.query.GetSQL()); } @@ -261,7 +271,8 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte scan.emplace_back(0, limit); maxIterations = limit; } - qres.AppendFront(OpAnd, SelectIterator{std::move(scan), false, "-scan", true}); + // Iterator Field Kind: -scan. Sorting Context! -> None + qres.AppendFront(OpAnd, SelectIterator{std::move(scan), false, "-scan", IteratorFieldKind::None, true}); } // Get maximum iterations count, for right calculation comparators costs qres.SortByCost(maxIterations); @@ -269,7 +280,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte // Check idset must be 1st qres.CheckFirstQuery(); - // Rewing all results iterators + // Rewind all results iterators qres.ExecuteAppropriateForEach(Skip{}, [reverse, maxIterations](SelectIterator &it) { it.Start(reverse, maxIterations); }); @@ -316,9 +327,9 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte // Get total count for simple query with 1 condition and 1 idset if (needCalcTotal && !lctx.calcTotal) { if (!ctx.query.entries.Empty()) { - result.totalCount = qres.Get(0).GetMaxIterations(); + result.totalCount += qres.Get(0).GetMaxIterations(); } else { - result.totalCount = ns_->items_.size() - ns_->free_.size(); + result.totalCount += ns_->items_.size() - ns_->free_.size(); } } explain.AddLoopTime(); @@ -343,16 +354,27 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte } } } - for (auto &aggregator : aggregators) { - result.aggregationResults.push_back(aggregator.GetResult()); + if (rx_unlikely(ctx.isMergeQuerySubQuery())) { + writeAggregationResultMergeSubQuery(result, aggregators, ctx); + } else { + for (auto &aggregator : aggregators) { + result.aggregationResults.push_back(aggregator.GetResult()); + } } // Put count/count_cached to aggretions - if (ctx.query.calcTotal != ModeNoTotal || containAggCount || containAggCountCached) { + if (aggregationQueryRef.calcTotal != ModeNoTotal || containAggCount || containAggCountCached) { AggregationResult ret; ret.fields = {"*"}; - ret.type = (ctx.query.calcTotal == ModeAccurateTotal || containAggCount) ? AggCount : AggCountCached; - ret.SetValue(result.totalCount); - result.aggregationResults.push_back(ret); + ret.type = (aggregationQueryRef.calcTotal == ModeAccurateTotal || containAggCount) ? AggCount : AggCountCached; + if (ctx.isMergeQuerySubQuery()) { + assertrx_throw(!result.aggregationResults.empty()); + auto &agg = result.aggregationResults.back(); + assertrx_throw(agg.type == ret.type); + agg.SetValue(result.totalCount); + } else { + ret.SetValue(result.totalCount); + result.aggregationResults.emplace_back(std::move(ret)); + } } explain.AddPostprocessTime(); @@ -365,7 +387,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte explain.PutSelectors(&qresHolder.GetResultsRef()); explain.PutJoinedSelectors(ctx.joinedSelectors); - if (rx_unlikely(ctx.query.debugLevel >= LogInfo)) { + if rx_unlikely (ctx.query.debugLevel >= LogInfo) { logPrintf(LogInfo, "%s", ctx.query.GetSQL()); explain.LogDump(ctx.query.debugLevel); } @@ -376,24 +398,24 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte result.explainResults = explain.GetJSON(); } } - if (rx_unlikely(ctx.query.debugLevel >= LogTrace)) { + if rx_unlikely (ctx.query.debugLevel >= LogTrace) { logPrintf(LogInfo, "Query returned: [%s]; total=%d", result.Dump(), result.totalCount); } if (needPutCachedTotal) { logPrintf(LogTrace, "[%s] put totalCount value into query cache: %d ", ns_->name_, result.totalCount); - ns_->queryTotalCountCache_->Put(ckey, {static_cast(result.totalCount)}); + ns_->queryTotalCountCache_->Put(ckey, {static_cast(result.totalCount - initTotalCount)}); } if (ctx.preResult && ctx.preResult->executionMode == JoinPreResult::ModeBuild) { switch (ctx.preResult->dataMode) { case JoinPreResult::ModeIdSet: - if (rx_unlikely(ctx.query.debugLevel >= LogInfo)) { + if rx_unlikely (ctx.query.debugLevel >= LogInfo) { logPrintf(LogInfo, "Built idset preResult (expected %d iterations) with %d ids, q = '%s'", explain.Iterations(), ctx.preResult->ids.size(), ctx.query.GetSQL()); } break; case JoinPreResult::ModeValues: - if (rx_unlikely(ctx.query.debugLevel >= LogInfo)) { + if rx_unlikely (ctx.query.debugLevel >= LogInfo) { logPrintf(LogInfo, "Built values preResult (expected %d iterations) with %d values, q = '%s'", explain.Iterations(), ctx.preResult->values.size(), ctx.query.GetSQL()); } @@ -1210,12 +1232,12 @@ void NsSelecter::checkStrictModeAgg(StrictMode strictMode, const std::string &na } } -h_vector NsSelecter::getAggregators(const Query &q) const { +h_vector NsSelecter::getAggregators(const std::vector &aggEntries, StrictMode strictMode) const { static constexpr int NotFilled = -2; h_vector ret; h_vector distinctIndexes; - for (const auto &ag : q.aggregations_) { + for (const auto &ag : aggEntries) { if (ag.Type() == AggCount || ag.Type() == AggCountCached) { continue; } @@ -1229,7 +1251,7 @@ h_vector NsSelecter::getAggregators(const Query &q) const { } int idx = -1; for (size_t i = 0; i < ag.Fields().size(); ++i) { - checkStrictModeAgg(q.strictMode == StrictModeNotSet ? ns_->config_.strictMode : q.strictMode, ag.Fields()[i], ns_->name_, + checkStrictModeAgg(strictMode == StrictModeNotSet ? ns_->config_.strictMode : strictMode, ag.Fields()[i], ns_->name_, ns_->tagsMatcher_); for (size_t j = 0; j < sortingEntries.size(); ++j) { @@ -1680,4 +1702,62 @@ bool NsSelecter::isSortOptimizatonEffective(const QueryEntries &qentries, Select return costOptimized <= costNormal; } +void NsSelecter::writeAggregationResultMergeSubQuery(QueryResults &result, h_vector &aggregators, SelectCtx &ctx) { + if (result.aggregationResults.size() < aggregators.size()) { + throw Error(errQueryExec, "Merged query(%s) aggregators count (%d) does not match to the parent query aggregations (%d)", + ctx.query.GetSQL(false), aggregators.size(), result.aggregationResults.size()); + } + for (size_t i = 0; i < aggregators.size(); i++) { + AggregationResult r = aggregators[i].GetResult(); + AggregationResult &parentRes = result.aggregationResults[i]; + if (r.type != parentRes.type || r.fields != parentRes.fields) { + std::stringstream strParentRes; + std::stringstream strR; + throw Error(errQueryExec, "Aggregation incorrect ns %s type of parent %s type of query %s parent field %s query field %s", + ns_->name_, AggTypeToStr(parentRes.type), AggTypeToStr(r.type), parentRes.DumpFields(strParentRes).str(), + r.DumpFields(strR).str()); + } + switch (r.type) { + case AggSum: { + std::optional newVal = r.GetValue(); + std::optional curVal = parentRes.GetValue(); + if (newVal.has_value()) { + double cur = 0.0; + if (curVal.has_value()) { + cur = curVal.value(); + } + parentRes.SetValue(newVal.value() + cur); + } + break; + } + case AggMin: { + std::optional newVal = r.GetValue(); + std::optional curVal = parentRes.GetValue(); + if (newVal.has_value()) { + if (!curVal.has_value() || newVal.value() < curVal.value()) { + parentRes.SetValue(newVal.value()); + } + } + break; + } + case AggMax: { + std::optional newVal = r.GetValue(); + std::optional curVal = parentRes.GetValue(); + if (newVal.has_value()) { + if (!curVal.has_value() || newVal.value() > curVal.value()) { + parentRes.SetValue(newVal.value()); + } + } + break; + } + case AggAvg: + case AggFacet: + case AggDistinct: + case AggCount: + case AggCountCached: + case AggUnknown: + assertrx_throw(false); + } + } +} } // namespace reindexer diff --git a/cpp_src/core/nsselecter/nsselecter.h b/cpp_src/core/nsselecter/nsselecter.h index 4b247ed6f..1a51ab7eb 100644 --- a/cpp_src/core/nsselecter/nsselecter.h +++ b/cpp_src/core/nsselecter/nsselecter.h @@ -17,6 +17,7 @@ struct SelectCtx { SelectFunctionsHolder *functions = nullptr; JoinPreResult::Ptr preResult; + ExplainCalc::Duration preResultTimeTotal = ExplainCalc::Duration::zero(); SortingContext sortingContext; uint8_t nsid = 0; bool isForceAll = false; @@ -31,6 +32,8 @@ struct SelectCtx { const Query *parentQuery = nullptr; ExplainCalc explain; bool requiresCrashTracking = false; + + RX_ALWAYS_INLINE bool isMergeQuerySubQuery() const noexcept { return isMergeQuery == IsMergeQuery::Yes && parentQuery; } }; class ItemComparator; @@ -77,7 +80,7 @@ class NsSelecter { void addSelectResult(uint8_t proc, IdType rowId, IdType properRowId, SelectCtx &sctx, h_vector &aggregators, QueryResults &result, bool preselectForFt); - h_vector getAggregators(const Query &) const; + h_vector getAggregators(const std::vector& aggEntrys, StrictMode strictMode) const; void setLimitAndOffset(ItemRefVector &result, size_t offset, size_t limit); void prepareSortingContext(SortingEntries &sortBy, SelectCtx &ctx, bool isFt, bool availableSelectBySortIndex); static void prepareSortIndex(const NamespaceImpl &, std::string_view column, int &index, bool &skipSortingEntry, StrictMode); @@ -97,6 +100,7 @@ class NsSelecter { void checkStrictModeAgg(StrictMode strictMode, const std::string &name, const std::string &nsName, const TagsMatcher &tagsMatcher) const; + void writeAggregationResultMergeSubQuery(QueryResults &result, h_vector &aggregators, SelectCtx &ctx); NamespaceImpl *ns_; SelectFunction::Ptr fnc_; FtCtx::Ptr ft_ctx_; diff --git a/cpp_src/core/nsselecter/querypreprocessor.cc b/cpp_src/core/nsselecter/querypreprocessor.cc index 19b14ba07..e7abaf49e 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.cc +++ b/cpp_src/core/nsselecter/querypreprocessor.cc @@ -1,4 +1,5 @@ #include "querypreprocessor.h" + #include "core/index/index.h" #include "core/index/indextext/indextext.h" #include "core/namespace/namespaceimpl.h" @@ -6,6 +7,7 @@ #include "core/nsselecter/selectiteratorcontainer.h" #include "core/nsselecter/sortexpression.h" #include "core/payload/fieldsset.h" +#include "core/query/dsl/dslencoder.h" #include "core/query/queryentry.h" #include "estl/overloaded.h" #include "nsselecter.h" @@ -119,6 +121,19 @@ void QueryPreprocessor::checkStrictMode(const std::string &index, int idxNo) con } } +class JoinOnExplainEnabled; +class JoinOnExplainDisabled; + +void QueryPreprocessor::InjectConditionsFromJoins(JoinedSelectors &js, OnConditionInjections &expalainOnInjections, + const RdxContext &rdxCtx) { + bool needExplain = query_.explain_ || query_.debugLevel >= LogInfo; + if (needExplain) { + injectConditionsFromJoins(0, container_.size(), js, expalainOnInjections, rdxCtx); + } else { + injectConditionsFromJoins(0, container_.size(), js, expalainOnInjections, rdxCtx); + } +} + void QueryPreprocessor::Reduce(bool isFt) { bool changed; do { @@ -356,12 +371,10 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si uint32_t maxSetSize = 0; for (auto i : res.entries) { auto &qe = Get(i); - if (rx_unlikely(!res.fields.contains(qe.idxNo))) { + if rx_unlikely (!res.fields.contains(qe.idxNo)) { throw Error(errLogic, "Error during composite index's fields substitution (this should not happen)"); } - if (rx_unlikely(qe.condition == CondEq && qe.values.size() == 0)) { - throw Error(errParams, "Condition EQ must have at least 1 argument, but provided 0"); - } + maxSetSize = std::max(maxSetSize, qe.values.size()); resultSetSize = (resultSetSize == 0) ? qe.values.size() : (resultSetSize * qe.values.size()); } @@ -499,7 +512,7 @@ bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { QueryEntry &rqe = Get(rhs); if ((lqe->condition == CondEq || lqe->condition == CondSet) && (rqe.condition == CondEq || rqe.condition == CondSet)) { // intersect 2 queryentries on the same index - if (rx_unlikely(lqe->values.empty())) { + if rx_unlikely (lqe->values.empty()) { return true; } if (container_[lhs].IsRef()) { @@ -581,9 +594,9 @@ void QueryPreprocessor::AddDistinctEntries(const h_vector &aggreg } } -void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, NamespaceImpl &rightNs, Query joinQuery, - std::string joinIndex, CondType condition, KeyValueType valuesType, - const RdxContext &rdxCtx) { +void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, std::string &explainStr, AggType &oAggType, + NamespaceImpl &rightNs, Query joinQuery, std::string joinIndex, CondType condition, + KeyValueType valuesType, const RdxContext &rdxCtx) { size_t limit; const auto &rNsCfg = rightNs.Config(); if (rNsCfg.maxPreselectSize == 0) { @@ -594,6 +607,7 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, Na limit = std::min(std::max(rNsCfg.minPreselectSize, rightNs.ItemsCount() * rNsCfg.maxPreselectPart), rNsCfg.maxPreselectSize); } + joinQuery.explain_ = query_.explain_; joinQuery.count = limit + 2; joinQuery.start = 0; joinQuery.sortingEntries_.clear(); @@ -603,14 +617,17 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, Na case CondEq: case CondSet: joinQuery.Distinct(std::move(joinIndex)); + oAggType = AggType::AggDistinct; break; case CondLt: case CondLe: joinQuery.Aggregate(AggMax, {std::move(joinIndex)}); + oAggType = AggType::AggMax; break; case CondGt: case CondGe: joinQuery.Aggregate(AggMin, {std::move(joinIndex)}); + oAggType = AggType::AggMin; break; case CondAny: case CondRange: @@ -620,11 +637,13 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, Na case CondDWithin: throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } + SelectCtx ctx{joinQuery, nullptr}; QueryResults qr; rightNs.Select(qr, ctx, rdxCtx); if (qr.Count() > limit) return; assertrx(qr.aggregationResults.size() == 1); + explainStr = qr.explainResults; switch (condition) { case CondEq: case CondSet: { @@ -725,23 +744,70 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, st } } -void QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, JoinedSelectors &js, const RdxContext &rdxCtx) { +template +void QueryPreprocessor::briefDump(size_t from, size_t to, const std::vector &joinedSelectors, WrSerializer &ser) const { + { + for (auto it = from; it < to; it = Next(it)) { + if (it != from || container_[it].operation != OpAnd) { + ser << container_[it].operation << ' '; + } + container_[it].InvokeAppropriate( + [&](const QueryEntriesBracket &b) { + ser << "("; + briefDump(it + 1, Next(it), joinedSelectors, ser); + dumpEqualPositions(0, ser, b.equalPositions); + ser << ")"; + }, + [&ser](const QueryEntry &qe) { ser << qe.DumpBrief() << ' '; }, + [&joinedSelectors, &ser](const JoinQueryEntry &jqe) { ser << jqe.Dump(joinedSelectors) << ' '; }, + [&ser](const BetweenFieldsQueryEntry &qe) { ser << qe.Dump() << ' '; }, + [&ser](const AlwaysFalse &) { ser << "AlwaysFalse" << ' '; }); + } + } +} + +template +size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, JoinedSelectors &js, OnConditionInjections &explainOnInjections, + const RdxContext &rdxCtx) { + using namespace std::string_view_literals; + + size_t injectedCount = 0; for (size_t cur = from; cur < to; cur = Next(cur)) { container_[cur].InvokeAppropriate( Skip{}, - [&js, cur, this, &rdxCtx](const QueryEntriesBracket &) { injectConditionsFromJoins(cur + 1, Next(cur), js, rdxCtx); }, + [&](const QueryEntriesBracket &) { + size_t injCount = injectConditionsFromJoins(cur + 1, Next(cur), js, explainOnInjections, rdxCtx); + to += injCount; + injectedCount += injCount; + assertrx_throw(to <= container_.size()); + }, [&](const JoinQueryEntry &jqe) { assertrx(js.size() > jqe.joinIndex); + JoinedSelector &joinedSelector = js[jqe.joinIndex]; - const bool byValues = joinedSelector.PreResult()->dataMode == JoinPreResult::ModeValues; + const bool byValues = joinedSelector.PreResult() && joinedSelector.PreResult()->dataMode == JoinPreResult::ModeValues; + + auto explainJoinOn = ExplainPolicy::AppendJoinOnExplain(explainOnInjections); + explainJoinOn.Init(jqe, js, byValues); + + // Checking if we are able to preselect something from RightNs, or there are preselected results if (!byValues) { const auto &rNsCfg = joinedSelector.RightNs()->Config(); - if (rNsCfg.maxPreselectSize == 0 && rNsCfg.maxPreselectPart == 0.0) return; + if (rNsCfg.maxPreselectSize == 0 && rNsCfg.maxPreselectPart == 0.0) { + explainJoinOn.Skipped("maxPreselectSize and maxPreselectPart == 0"sv); + return; + } } else { - if (!joinedSelector.PreResult()->values.IsPreselectAllowed()) return; + if (!joinedSelector.PreResult()->values.IsPreselectAllowed()) { + explainJoinOn.Skipped("Preselect is not allowed"sv); + return; + } } - assertrx(joinedSelector.Type() == InnerJoin || joinedSelector.Type() == OrInnerJoin); + const auto &joinEntries = joinedSelector.joinQuery_.joinEntries_; + // LeftJoin-s shall not be in QueryEntries container_ by construction + assertrx(joinedSelector.Type() == InnerJoin || joinedSelector.Type() == OrInnerJoin); + // Checking if we have anything to inject into main Where clause bool foundANDOrOR = false; for (const auto &je : joinEntries) { if (je.op_ != OpNot) { @@ -749,21 +815,35 @@ void QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Joined break; } } - if (!foundANDOrOR) return; + if (!foundANDOrOR) { + explainJoinOn.Skipped("And or Or operators not found"sv); + return; + } + OpType op = GetOperation(cur); if (joinedSelector.Type() == OrInnerJoin) { if (op == OpNot) throw Error(errParams, "OR INNER JOIN with operation NOT"); op = OpOr; joinedSelector.SetType(InnerJoin); } + + // inserting Bracket for JoinQuery itself into ExpressionTree SetOperation(OpAnd, cur); + // !!!Warning jqe reference will be invalidated after EncloseInBracket EncloseInBracket(cur, cur + 1, op); ++cur; + + explainJoinOn.ReserveOnEntries(joinEntries.size()); + size_t count = 0; bool prevIsSkipped = false; size_t orChainLength = 0; for (size_t i = 0, s = joinEntries.size(); i < s; ++i) { const QueryJoinEntry &joinEntry = joinEntries[i]; + + auto explainEntry = explainJoinOn.AppendOnEntryExplain(); + explainEntry.InitialCondition(joinEntry, joinedSelector); + CondType condition = joinEntry.condition_; OpType operation = joinEntry.op_; switch (operation) { @@ -785,6 +865,7 @@ void QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Joined case CondEq: case CondSet: prevIsSkipped = true; + explainEntry.Skipped("Skipped due to condition Eq|Set with operation Not."sv); continue; case CondAny: case CondRange: @@ -797,13 +878,17 @@ void QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Joined operation = OpAnd; break; case OpOr: - if (prevIsSkipped) continue; + explainEntry.OrChainPart(true); + if (prevIsSkipped) { + continue; + } ++orChainLength; break; case OpAnd: orChainLength = 0; break; } + QueryEntry newEntry; newEntry.index = joinEntry.index_; newEntry.idxNo = IndexValueType::SetByJsonPath; @@ -814,6 +899,7 @@ void QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Joined valuesType = index.SelectKeyType(); collate = index.Opts().collateOpts_; } + if (byValues) { assertrx(joinedSelector.itemQuery_.entries.HoldsOrReferTo(i)); const QueryEntry &qe = joinedSelector.itemQuery_.entries.Get(i); @@ -835,6 +921,9 @@ void QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Joined case CondGe: { const QueryEntry &qe = joinedSelector.itemQuery_.entries.Get(i); skip = qe.idxNo != IndexValueType::SetByJsonPath && joinedSelector.RightNs()->indexes_[qe.idxNo]->IsUuid(); + if (skip) { + explainEntry.Skipped("Skipped due to condition Lt|Le|Gt|Ge with UUID index field."sv); + } break; } case CondEq: @@ -848,30 +937,157 @@ void QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Joined break; } if (!skip) { - fillQueryEntryFromOnCondition(newEntry, *joinedSelector.RightNs(), joinedSelector.JoinQuery(), - joinEntry.joinIndex_, condition, valuesType, rdxCtx); + std::string explainSelect; + AggType selectAggType; + fillQueryEntryFromOnCondition(newEntry, explainSelect, selectAggType, *joinedSelector.RightNs(), + joinedSelector.JoinQuery(), joinEntry.joinIndex_, condition, valuesType, rdxCtx); + explainEntry.ExplainSelect(std::move(explainSelect), selectAggType); } } + if (!newEntry.values.empty()) { + explainEntry.Succeed(newEntry); + Insert(cur, operation, std::move(newEntry)); ++cur; ++count; prevIsSkipped = false; } else { + explainEntry.Skipped("Skipped as cannot obtain values from right namespace."sv); if (operation == OpOr) { Erase(cur - orChainLength, cur); count -= orChainLength; + // Marking On-injections as fail for removed entries. + explainJoinOn.FailOnEntriesAsOrChain(orChainLength); } prevIsSkipped = true; } - } + } // end of entries processing + if (count > 0) { EncloseInBracket(cur - count, cur, OpAnd); + + explainJoinOn.Succeed( + [this, cur, count, &js](WrSerializer &ser) { briefDump(cur - count, Next(cur - count), js, ser); }); + ++cur; + injectedCount += count + 2; to += count + 2; } }); } + return injectedCount; } +class JoinOnExplainDisabled { + JoinOnExplainDisabled() noexcept = default; + struct OnEntryExplain { + OnEntryExplain() noexcept = default; + + RX_ALWAYS_INLINE void InitialCondition(const QueryJoinEntry &, const JoinedSelector &) const noexcept {} + RX_ALWAYS_INLINE void Succeed(const QueryEntry &) const noexcept {} + RX_ALWAYS_INLINE void Skipped(std::string_view) const noexcept {} + RX_ALWAYS_INLINE void OrChainPart(bool) const noexcept {} + RX_ALWAYS_INLINE void ExplainSelect(std::string &&, AggType) const noexcept {} + }; + +public: + [[nodiscard]] RX_ALWAYS_INLINE static JoinOnExplainDisabled AppendJoinOnExplain(OnConditionInjections &) noexcept { return {}; } + + RX_ALWAYS_INLINE void Init(const JoinQueryEntry &, const JoinedSelectors &, bool) const noexcept {} + RX_ALWAYS_INLINE void Succeed(const std::function &) const noexcept {} + RX_ALWAYS_INLINE void Skipped(std::string_view) const noexcept {} + RX_ALWAYS_INLINE void ReserveOnEntries(size_t) const noexcept {} + [[nodiscard]] RX_ALWAYS_INLINE OnEntryExplain AppendOnEntryExplain() const noexcept { return {}; } + + RX_ALWAYS_INLINE void FailOnEntriesAsOrChain(size_t) const noexcept {} +}; + +class JoinOnExplainEnabled { + using time_point_t = ExplainCalc::Clock::time_point; + struct OnEntryExplain { + OnEntryExplain(ConditionInjection &explainEntry) noexcept : startTime_(ExplainCalc::Clock::now()), explainEntry_(explainEntry) {} + ~OnEntryExplain() noexcept { explainEntry_.totalTime_ = ExplainCalc::Clock::now() - startTime_; } + OnEntryExplain(const OnEntryExplain &) = delete; + OnEntryExplain(OnEntryExplain &&) = delete; + OnEntryExplain &operator=(const OnEntryExplain &) = delete; + OnEntryExplain &operator=(OnEntryExplain &&) = delete; + + void InitialCondition(const QueryJoinEntry &joinEntry, const JoinedSelector &joinedSelector) { + explainEntry_.initCond = joinEntry.DumpCondition(joinedSelector); + } + void Succeed(const QueryEntry &newEntry) { + explainEntry_.succeed = true; + explainEntry_.reason = ""; + explainEntry_.newCond = newEntry.DumpBrief(); + explainEntry_.valuesCount = newEntry.values.size(); + } + + void Skipped(std::string_view reason) noexcept { + if (explainEntry_.reason.empty()) { + explainEntry_.reason = reason; + } + explainEntry_.succeed = false; + } + + void OrChainPart(bool orChainPart) noexcept { explainEntry_.orChainPart_ = orChainPart; } + void ExplainSelect(std::string &&explain, AggType aggType) noexcept { + explainEntry_.explain = std::move(explain); + explainEntry_.aggType = aggType; + } + + private: + time_point_t startTime_; + ConditionInjection &explainEntry_; + }; + + JoinOnExplainEnabled(const JoinOnExplainEnabled &) = delete; + JoinOnExplainEnabled(JoinOnExplainEnabled &&) = delete; + JoinOnExplainEnabled &operator=(const JoinOnExplainEnabled &) = delete; + JoinOnExplainEnabled &operator=(JoinOnExplainEnabled &&) = delete; + + JoinOnExplainEnabled(JoinOnInjection &joinOn) noexcept : explainJoinOn_(joinOn), startTime_(ExplainCalc::Clock::now()) {} + +public: + [[nodiscard]] static JoinOnExplainEnabled AppendJoinOnExplain(OnConditionInjections &explainOnInjections) { + return {explainOnInjections.emplace_back()}; + } + ~JoinOnExplainEnabled() noexcept { explainJoinOn_.totalTime_ = ExplainCalc::Clock::now() - startTime_; } + + void Init(const JoinQueryEntry &jqe, const JoinedSelectors &js, bool byValues) { + const JoinedSelector &joinedSelector = js[jqe.joinIndex]; + explainJoinOn_.rightNsName = joinedSelector.RightNsName(); + explainJoinOn_.joinCond = jqe.DumpOnCondition(js); + explainJoinOn_.type = byValues ? JoinOnInjection::ByValue : JoinOnInjection::Select; + } + void Succeed(const std::function &setInjectedCond) { + explainJoinOn_.succeed = true; + setInjectedCond(explainJoinOn_.injectedCond); + } + void Skipped(std::string_view reason) noexcept { + if (explainJoinOn_.reason.empty()) { + explainJoinOn_.reason = reason; + } + explainJoinOn_.succeed = false; + } + + void ReserveOnEntries(size_t count) { explainJoinOn_.conditions.reserve(count); } + [[nodiscard]] OnEntryExplain AppendOnEntryExplain() { return {explainJoinOn_.conditions.emplace_back()}; }; + + void FailOnEntriesAsOrChain(size_t orChainLength) { + using namespace std::string_view_literals; + auto &conditions = explainJoinOn_.conditions; + assertrx(conditions.size() >= orChainLength); + // Marking On-injections as fail for removed entries. + for (size_t jsz = conditions.size(), j = jsz - orChainLength; j < jsz; ++j) { + conditions[j].succeed = false; + conditions[j].orChainPart_ = true; + } + } + +private: + JoinOnInjection &explainJoinOn_; + time_point_t startTime_; +}; + } // namespace reindexer diff --git a/cpp_src/core/nsselecter/querypreprocessor.h b/cpp_src/core/nsselecter/querypreprocessor.h index 31d3dbd29..907636251 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.h +++ b/cpp_src/core/nsselecter/querypreprocessor.h @@ -47,13 +47,12 @@ class QueryPreprocessor : private QueryEntries { unsigned Count() const noexcept { return count_; } bool MoreThanOneEvaluation() const noexcept { return queryEntryAddedByForcedSortOptimization_; } bool AvailableSelectBySortIndex() const noexcept { return !queryEntryAddedByForcedSortOptimization_ || !forcedStage(); } - void InjectConditionsFromJoins(JoinedSelectors &js, const RdxContext &rdxCtx) { - injectConditionsFromJoins(0, container_.size(), js, rdxCtx); - } + void InjectConditionsFromJoins(JoinedSelectors &js, OnConditionInjections &expalainOnInjections, const RdxContext &rdxCtx); void Reduce(bool isFt); void InitIndexNumbers(); using QueryEntries::Size; using QueryEntries::Dump; + using QueryEntries::ToDsl; [[nodiscard]] SortingEntries GetSortingEntries(const SelectCtx &ctx) const; bool IsFtExcluded() const noexcept { return ftEntry_.has_value(); } void ExcludeFtQuery(const RdxContext &); @@ -90,9 +89,13 @@ class QueryPreprocessor : private QueryEntries { [[nodiscard]] const Index *findMaxIndex(QueryEntries::const_iterator begin, QueryEntries::const_iterator end) const; void findMaxIndex(QueryEntries::const_iterator begin, QueryEntries::const_iterator end, h_vector &foundIndexes) const; - void injectConditionsFromJoins(size_t from, size_t to, JoinedSelectors &, const RdxContext &); - void fillQueryEntryFromOnCondition(QueryEntry &, NamespaceImpl &rightNs, Query joinQuery, std::string joinIndex, CondType condition, - KeyValueType, const RdxContext &); + /** @brief recurrently checks and injects Join ON conditions + * @returns injected conditions and EntryBrackets count + */ + template + size_t injectConditionsFromJoins(size_t from, size_t to, JoinedSelectors &, OnConditionInjections &, const RdxContext &); + void fillQueryEntryFromOnCondition(QueryEntry &, std::string &outExplainStr, AggType &, NamespaceImpl &rightNs, Query joinQuery, + std::string joinIndex, CondType condition, KeyValueType, const RdxContext &); template void fillQueryEntryFromOnCondition(QueryEntry &, std::string_view joinIndex, CondType condition, const JoinedSelector &, KeyValueType, int rightIdxNo, const CollateOpts &); @@ -101,6 +104,9 @@ class QueryPreprocessor : private QueryEntries { size_t removeBrackets(size_t begin, size_t end); bool canRemoveBracket(size_t i) const; + template + void briefDump(size_t from, size_t to, const std::vector &joinedSelectors, WrSerializer &ser) const; + NamespaceImpl &ns_; const Query &query_; StrictMode strictMode_; diff --git a/cpp_src/core/nsselecter/selectiterator.cc b/cpp_src/core/nsselecter/selectiterator.cc index 9bc60dd4d..05ab657be 100644 --- a/cpp_src/core/nsselecter/selectiterator.cc +++ b/cpp_src/core/nsselecter/selectiterator.cc @@ -1,13 +1,14 @@ #include "selectiterator.h" + #include #include #include "core/index/indexiterator.h" namespace reindexer { -SelectIterator::SelectIterator(SelectKeyResult res, bool dist, std::string n, bool forcedFirst) - : SelectKeyResult(std::move(res)), distinct(dist), name(std::move(n)), forcedFirst_(forcedFirst), type_(Forward) {} +SelectIterator::SelectIterator(SelectKeyResult res, bool dist, std::string n, IteratorFieldKind fKind, bool forcedFirst) + : SelectKeyResult(std::move(res)), distinct(dist), name(std::move(n)), fieldKind(fKind), forcedFirst_(forcedFirst), type_(Forward) {} void SelectIterator::Bind(const PayloadType &type, int field) { for (Comparator &cmp : comparators_) cmp.Bind(type, field); diff --git a/cpp_src/core/nsselecter/selectiterator.h b/cpp_src/core/nsselecter/selectiterator.h index 0bba0546c..55286e183 100644 --- a/cpp_src/core/nsselecter/selectiterator.h +++ b/cpp_src/core/nsselecter/selectiterator.h @@ -4,6 +4,7 @@ namespace reindexer { +enum class IteratorFieldKind { None, NonIndexed, Indexed }; /// Allows to iterate over a result of selecting /// data for one certain key. class SelectIterator : public SelectKeyResult { @@ -23,7 +24,7 @@ class SelectIterator : public SelectKeyResult { }; SelectIterator() = default; - SelectIterator(SelectKeyResult res, bool distinct, std::string name, bool forcedFirst = false); + SelectIterator(SelectKeyResult res, bool distinct, std::string name, IteratorFieldKind fieldKind, bool forcedFirst = false); /// Starts iteration process: prepares /// object for further work. @@ -135,6 +136,7 @@ class SelectIterator : public SelectKeyResult { bool distinct = false; std::string name; + IteratorFieldKind fieldKind; protected: // Iterates to a next item of result diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.cc b/cpp_src/core/nsselecter/selectiteratorcontainer.cc index 48ee368ab..a926d6fb9 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.cc +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.cc @@ -1,4 +1,5 @@ #include "selectiteratorcontainer.h" + #include #include #include "core/namespace/namespaceimpl.h" @@ -189,6 +190,34 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe FieldsSet fields; TagsPath tagsPath = ns.tagsMatcher_.path2tag(qe.index); + + // TODO: it may be necessary to remove or change this switch after QueryEntry refactoring + switch (qe.condition) { + case CondAny: + case CondEmpty: + case CondAllSet: + case CondEq: + case CondSet: + break; + case CondRange: + case CondDWithin: + if (qe.values.size() != 2) { + throw Error(errParams, "For condition %s required exactly 2 arguments, but provided %d", CondTypeToStr(qe.condition), + qe.values.size()); + } + break; + case CondLt: + case CondLe: + case CondGt: + case CondGe: + case CondLike: + if (qe.values.size() != 1) { + throw Error(errParams, "For condition %s required exactly 1 argument, but provided %d", CondTypeToStr(qe.condition), + qe.values.size()); + } + break; + } + if (!tagsPath.empty()) { SelectKeyResult comparisonResult; fields.push_back(tagsPath); @@ -330,7 +359,9 @@ void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectR [[fallthrough]]; case OpNot: case OpAnd: - Append(op, SelectIterator(res, qe.distinct, qe.index, isIndexFt)); + // Iterator Field Kind: Query entry results. Field known. + Append(op, SelectIterator(res, qe.distinct, qe.index, + qe.idxNo < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed, isIndexFt)); if (!nonIndexField && !isIndexSparse) { // last appended is always a SelectIterator const auto lastAppendedIt = lastAppendedOrClosed(); diff --git a/cpp_src/core/nsselecter/substitutionhelpers.h b/cpp_src/core/nsselecter/substitutionhelpers.h index c7ade04ff..f53fee54e 100644 --- a/cpp_src/core/nsselecter/substitutionhelpers.h +++ b/cpp_src/core/nsselecter/substitutionhelpers.h @@ -10,7 +10,7 @@ namespace composite_substitution_helpers { class CompositeValuesCountLimits { public: uint32_t operator[](uint32_t fieldsCount) const noexcept { - if (rx_unlikely(fieldsCount >= limits_.size())) { + if rx_unlikely (fieldsCount >= limits_.size()) { return kMaxValuesCount; } return limits_[fieldsCount]; diff --git a/cpp_src/core/payload/fieldsset.h b/cpp_src/core/payload/fieldsset.h index b04f62035..87da7eefe 100644 --- a/cpp_src/core/payload/fieldsset.h +++ b/cpp_src/core/payload/fieldsset.h @@ -20,6 +20,8 @@ static_assert(std::numeric_limits::min() <= SetByJs static_assert(sizeof(std::bitset) == 32, "Expecting no overhead from std::bitset"); using FieldsPath = std::variant; +using ScalarIndexesSetT = std::bitset; + class IndexesFieldsSet { public: IndexesFieldsSet() noexcept = default; diff --git a/cpp_src/core/payload/payloadiface.cc b/cpp_src/core/payload/payloadiface.cc index 6fe4f9e8a..110d0b80f 100644 --- a/cpp_src/core/payload/payloadiface.cc +++ b/cpp_src/core/payload/payloadiface.cc @@ -93,7 +93,7 @@ void PayloadIface::GetByJsonPath(const IndexedTagsPath &tagsPath, VariantArra ConstPayload pl(t_, *v_); FieldsSet filter({tagsPath}); BaseEncoder encoder(nullptr, &filter); - krefs.clear(); + krefs.Clear(); if (!tagsPath.empty()) { FieldsExtractor extractor(&krefs, expectedType, tagsPath.size(), &filter); encoder.Encode(pl, extractor); diff --git a/cpp_src/core/query/dsl/dslencoder.cc b/cpp_src/core/query/dsl/dslencoder.cc index 3e0404158..3561d5b24 100644 --- a/cpp_src/core/query/dsl/dslencoder.cc +++ b/cpp_src/core/query/dsl/dslencoder.cc @@ -1,4 +1,5 @@ #include "dslencoder.h" + #include #include "core/cjson/jsonbuilder.h" #include "core/keyvalue/key_string.h" diff --git a/cpp_src/core/query/dsl/dslencoder.h b/cpp_src/core/query/dsl/dslencoder.h index 55928152b..b629d74b5 100644 --- a/cpp_src/core/query/dsl/dslencoder.h +++ b/cpp_src/core/query/dsl/dslencoder.h @@ -8,5 +8,6 @@ class Query; namespace dsl { std::string toDsl(const Query& query); -} +} // namespace dsl + } // namespace reindexer diff --git a/cpp_src/core/query/dsl/dslparser.cc b/cpp_src/core/query/dsl/dslparser.cc index d5c847fd4..f5fbfc709 100644 --- a/cpp_src/core/query/dsl/dslparser.cc +++ b/cpp_src/core/query/dsl/dslparser.cc @@ -327,7 +327,6 @@ static void parseFilter(const JsonValue& filter, Query& q, std::vector())) + throw Error(errParseDSL, "The array \"values\" must contain only a string type value for the type \"expression\""); + if (isObject) { - query.SetObject(fieldName, values); + query.SetObject(fieldName, std::move(values)); } else { - query.Set(fieldName, values, isExpression); + query.Set(fieldName, std::move(values), isExpression); } } } diff --git a/cpp_src/core/query/expressionevaluator.cc b/cpp_src/core/query/expressionevaluator.cc index d172b8666..8112d8c9c 100644 --- a/cpp_src/core/query/expressionevaluator.cc +++ b/cpp_src/core/query/expressionevaluator.cc @@ -3,143 +3,222 @@ #include "core/payload/payloadiface.h" #include "core/selectfunc/functionexecutor.h" #include "core/selectfunc/selectfunc.h" +#include "double-conversion/double-conversion.h" #include "estl/tokenizer.h" namespace reindexer { using namespace std::string_view_literals; -const char* kWrongFieldTypeError = "Only integral type non-array fields are supported in arithmetical expressions: %s"; - -ExpressionEvaluator::ExpressionEvaluator(const PayloadType& type, TagsMatcher& tagsMatcher, FunctionExecutor& func) - : type_(type), tagsMatcher_(tagsMatcher), functionExecutor_(func) {} +constexpr char kWrongFieldTypeError[] = "Only integral type non-array fields are supported in arithmetical expressions: %s"; +constexpr char kScalarsInConcatenationError[] = "Unable to use scalar values in the arrays concatenation expressions: %s"; void ExpressionEvaluator::captureArrayContent(tokenizer& parser) { - token tok = parser.next_token(false); - for (;;) { - tok = parser.next_token(false); - if (tok.text() == "]"sv) { - if (arrayValues_.empty()) break; + arrayValues_.MarkArray(); + token tok = parser.next_token(tokenizer::flags::no_flags); + if (tok.text() == "]") { + return; + } + for (;; tok = parser.next_token(tokenizer::flags::no_flags)) { + if rx_unlikely (tok.text() == "]"sv) { throw Error(errParseSQL, "Expected field value, but found ']' in query, %s", parser.where()); } arrayValues_.emplace_back(token2kv(tok, parser, false)); - tok = parser.next_token(); - if (tok.text() == "]"sv) break; - if (tok.text() != ","sv) { + tok = parser.next_token(tokenizer::flags::no_flags); + if (tok.text() == "]"sv) { + break; + } + if rx_unlikely (tok.text() != ","sv) { throw Error(errParseSQL, "Expected ']' or ',', but found '%s' in query, %s", tok.text(), parser.where()); } - }; + } } -double ExpressionEvaluator::getPrimaryToken(tokenizer& parser, const PayloadValue& v) { - token tok = parser.peek_token(true, true); - if (tok.text() == "("sv) { - parser.next_token(); - double val = performSumAndSubtracting(parser, v); - if (parser.next_token().text() != ")"sv) throw Error(errLogic, "')' expected in arithmetical expression"); - return val; - } else if (tok.text() == "["sv) { +void ExpressionEvaluator::throwUnexpectedTokenError(tokenizer& parser, const token& outTok) { + if (state_ == StateArrayConcat || parser.peek_token(tokenizer::flags::treat_sign_as_token).text() == "|"sv) { + throw Error(errParams, kScalarsInConcatenationError, outTok.text()); + } + throw Error(errParams, kWrongFieldTypeError, outTok.text()); +} + +ExpressionEvaluator::PrimaryToken ExpressionEvaluator::getPrimaryToken(tokenizer& parser, const PayloadValue& v, token& outTok) { + outTok = parser.next_token(); + if (outTok.text() == "("sv) { + const double val = performSumAndSubtracting(parser, v); + if rx_unlikely (parser.next_token().text() != ")"sv) { + throw Error(errParams, "')' expected in arithmetical expression"); + } + return {.value = val, .type = PrimaryToken::Type::Scalar}; + } else if (outTok.text() == "["sv) { captureArrayContent(parser); - } else if (tok.type == TokenNumber) { - char* p = nullptr; - parser.next_token(); - return strtod(tok.text().data(), &p); - } else if (tok.type == TokenName) { - int field = 0; - VariantArray fieldValues; - ConstPayload pv(type_, v); - if (type_.FieldByName(tok.text(), field)) { - const auto type = type_.Field(field).Type(); - if (type_.Field(field).IsArray()) { - pv.Get(field, fieldValues); - for (const Variant& v : fieldValues) { - arrayValues_.emplace_back(v); - } - parser.next_token(); - return 0.0; - } else if (state_ == StateArrayConcat) { - VariantArray vals; - pv.GetByJsonPath(tok.text(), tagsMatcher_, vals, KeyValueType::Undefined{}); - for (const Variant& v : vals) { - arrayValues_.emplace_back(v); - } - parser.next_token(); - return 0.0; - } else { - return type.EvaluateOneOf( - [&](OneOf) { - pv.Get(field, fieldValues); - if (fieldValues.empty()) throw Error(errLogic, "Calculating value of an empty field is impossible: %s", tok.text()); - parser.next_token(); - return fieldValues.front().As(); - }, - [&](OneOf) -> double { - throw Error(errLogic, kWrongFieldTypeError, tok.text()); - }, - [](OneOf) noexcept -> double { - assertrx(0); - abort(); - }); + return {.value = std::nullopt, .type = PrimaryToken::Type::Array}; + } + switch (outTok.type) { + case TokenNumber: { + try { + using double_conversion::StringToDoubleConverter; + static const StringToDoubleConverter converter{StringToDoubleConverter::NO_FLAGS, NAN, NAN, nullptr, nullptr}; + int countOfCharsParsedAsDouble; + return {.value = converter.StringToDouble(outTok.text_.data(), outTok.text_.size(), &countOfCharsParsedAsDouble), + .type = PrimaryToken::Type::Scalar}; + } catch (...) { + throw Error(errParams, "Unable to convert '%s' to double value", outTok.text()); } - } else { - pv.GetByJsonPath(tok.text(), tagsMatcher_, fieldValues, KeyValueType::Undefined{}); - if (fieldValues.size() > 0) { - const auto type = fieldValues.front().Type(); - if ((fieldValues.size() > 1) || (state_ == StateArrayConcat)) { - for (const Variant& v : fieldValues) { - arrayValues_.emplace_back(v); - } - parser.next_token(); - return 0.0; - } else { - return type.EvaluateOneOf( - [&](OneOf) { - parser.next_token(); - return fieldValues.front().As(); - }, - [&](OneOf) -> double { - throw Error(errLogic, kWrongFieldTypeError, tok.text()); - }); - } - } else { - SelectFuncStruct funcData = SelectFuncParser().ParseFunction(parser, true); - funcData.field = forField_; - return functionExecutor_.Execute(funcData).As(); + } + case TokenName: + return handleTokenName(parser, v, outTok); + case TokenString: + throwUnexpectedTokenError(parser, outTok); + case TokenEnd: + case TokenOp: + case TokenSymbol: + case TokenSign: + break; + } + throw Error(errParams, "Unexpected token in expression: '%s'", outTok.text()); +} + +ExpressionEvaluator::PrimaryToken ExpressionEvaluator::handleTokenName(tokenizer& parser, const PayloadValue& v, token& outTok) { + int field = 0; + VariantArray fieldValues; + ConstPayload pv(type_, v); + if (type_.FieldByName(outTok.text(), field)) { + if (type_.Field(field).IsArray()) { + pv.Get(field, fieldValues); + arrayValues_.MarkArray(); + for (Variant& v : fieldValues) { + arrayValues_.emplace_back(std::move(v)); } + return (state_ == StateArrayConcat || fieldValues.size() != 1) + ? PrimaryToken{.value = std::nullopt, .type = PrimaryToken::Type::Array} + : type_.Field(field).Type().EvaluateOneOf( + [this](OneOf) -> PrimaryToken { + return {.value = arrayValues_.back().As(), .type = PrimaryToken::Type::Array}; + }, + [&, this](OneOf) -> PrimaryToken { + if rx_unlikely (state_ != StateArrayConcat && + parser.peek_token(tokenizer::flags::treat_sign_as_token).text() != "|"sv) { + throw Error(errParams, kWrongFieldTypeError, outTok.text()); + } + return {.value = std::nullopt, .type = PrimaryToken::Type::Array}; + }, + [](OneOf) noexcept + -> PrimaryToken { + assertrx_throw(false); + abort(); + }); } - } else { - throw Error(errLogic, "Only integral type non-array fields are supported in arithmetical expressions"); + return type_.Field(field).Type().EvaluateOneOf( + [&](OneOf) -> PrimaryToken { + pv.Get(field, fieldValues); + if rx_unlikely (fieldValues.empty()) { + throw Error(errParams, "Calculating value of an empty field is impossible: %s", outTok.text()); + } + return {.value = fieldValues.front().As(), .type = PrimaryToken::Type::Scalar}; + }, + [&, this](OneOf) -> PrimaryToken { + throwUnexpectedTokenError(parser, outTok); + }, + [](OneOf) -> PrimaryToken { + assertrx_throw(false); + abort(); + }); + } else if rx_unlikely (outTok.text() == "true"sv || outTok.text() == "false"sv) { + throwUnexpectedTokenError(parser, outTok); } - return 0.0; + + pv.GetByJsonPath(outTok.text(), tagsMatcher_, fieldValues, KeyValueType::Undefined{}); + + if (fieldValues.IsNullValue()) { + return {.value = std::nullopt, .type = PrimaryToken::Type::Null}; + } + + const bool isArrayField = fieldValues.IsArrayValue(); + if (isArrayField) { + for (Variant& v : fieldValues) { + arrayValues_.emplace_back(std::move(v)); + } + if ((state_ == StateArrayConcat) || (fieldValues.size() != 1)) { + return {.value = std::nullopt, .type = PrimaryToken::Type::Array}; + } + } + if (fieldValues.size() == 1) { + const Variant* vptr = isArrayField ? &arrayValues_.back() : &fieldValues.front(); + return vptr->Type().EvaluateOneOf( + [vptr, isArrayField](OneOf) -> PrimaryToken { + return {.value = vptr->As(), .type = isArrayField ? PrimaryToken::Type::Array : PrimaryToken::Type::Scalar}; + }, + [&, this](OneOf) -> PrimaryToken { + if (isArrayField) { + return {.value = std::nullopt, .type = PrimaryToken::Type::Array}; + } + throwUnexpectedTokenError(parser, outTok); + }, + [](OneOf) -> PrimaryToken { + assertrx_throw(0); + abort(); + }); + } else if (parser.peek_token(tokenizer::flags::treat_sign_as_token).text() == "(") { + SelectFuncStruct funcData = SelectFuncParser().ParseFunction(parser, true, outTok); + funcData.field = std::string(forField_); + return {.value = functionExecutor_.Execute(funcData).As(), .type = PrimaryToken::Type::Scalar}; + } + return {.value = std::nullopt, .type = PrimaryToken::Type::Null}; } double ExpressionEvaluator::performArrayConcatenation(tokenizer& parser, const PayloadValue& v, token& tok) { - double left = getPrimaryToken(parser, v); + token valueToken; + auto left = getPrimaryToken(parser, v, valueToken); tok = parser.peek_token(); + switch (left.type) { + case PrimaryToken::Type::Scalar: + if rx_unlikely (tok.text() == "|"sv) { + throw Error(errParams, kScalarsInConcatenationError, valueToken.text()); + } + break; + case PrimaryToken::Type::Array: + case PrimaryToken::Type::Null: + if rx_unlikely (!left.value.has_value() && tok.text() != "|"sv) { + throw Error(errParams, "Unable to use array and null values outside of the arrays concatenation"); + } + break; + } + while (tok.text() == "|"sv) { parser.next_token(); tok = parser.next_token(); - if (tok.text() != "|") throw Error(errLogic, "Expected '|', not %s", tok.text()); + if rx_unlikely (tok.text() != "|") { + throw Error(errParams, "Expected '|', not %s", tok.text()); + } + if rx_unlikely (state_ != StateArrayConcat && state_ != None) { + throw Error(errParams, "Unable to mix arrays concatenation and arithmetic operations. Got token: '%s'", tok.text()); + } state_ = StateArrayConcat; - getPrimaryToken(parser, v); + const auto right = getPrimaryToken(parser, v, valueToken); + if rx_unlikely (right.type == PrimaryToken::Type::Scalar) { + throw Error(errParams, kScalarsInConcatenationError, valueToken.text()); + } + assertrx_throw(!right.value.has_value()); tok = parser.peek_token(); } - return left; + return left.value.has_value() ? left.value.value() : 0.0; } double ExpressionEvaluator::performMultiplicationAndDivision(tokenizer& parser, const PayloadValue& v, token& tok) { double left = performArrayConcatenation(parser, v, tok); - tok = parser.peek_token(true, true); + tok = parser.peek_token(tokenizer::flags::treat_sign_as_token); while (tok.text() == "*"sv || tok.text() == "/"sv) { + if rx_unlikely (state_ == StateArrayConcat) { + throw Error(errParams, "Unable to mix arrays concatenation and arithmetic operations. Got token: '%s'", tok.text()); + } state_ = StateMultiplyAndDivide; if (tok.text() == "*"sv) { - parser.next_token(); + parser.next_token(tokenizer::flags::treat_sign_as_token); left *= performMultiplicationAndDivision(parser, v, tok); - } else if (tok.text() == "/"sv) { - parser.next_token(); - double val = performMultiplicationAndDivision(parser, v, tok); + } else { + // tok.text() == "/"sv + parser.next_token(tokenizer::flags::treat_sign_as_token); + const double val = performMultiplicationAndDivision(parser, v, tok); if (val == 0) throw Error(errLogic, "Division by zero!"); left /= val; } @@ -150,35 +229,31 @@ double ExpressionEvaluator::performMultiplicationAndDivision(tokenizer& parser, double ExpressionEvaluator::performSumAndSubtracting(tokenizer& parser, const PayloadValue& v) { token tok; double left = performMultiplicationAndDivision(parser, v, tok); - tok = parser.peek_token(true, true); + tok = parser.peek_token(tokenizer::flags::treat_sign_as_token); while (tok.text() == "+"sv || tok.text() == "-"sv) { + if rx_unlikely (state_ == StateArrayConcat) { + throw Error(errParams, "Unable to mix arrays concatenation and arithmetic operations. Got token: '%s'", tok.text()); + } state_ = StateSumAndSubtract; if (tok.text() == "+"sv) { - parser.next_token(true, true); + parser.next_token(tokenizer::flags::treat_sign_as_token); left += performMultiplicationAndDivision(parser, v, tok); - } else if (tok.text() == "-"sv) { - parser.next_token(true, true); + } else { + // tok.text() == "-"sv + parser.next_token(tokenizer::flags::treat_sign_as_token); left -= performMultiplicationAndDivision(parser, v, tok); } } return left; } -VariantArray ExpressionEvaluator::Evaluate(tokenizer& parser, const PayloadValue& v, std::string_view forField) { - forField_ = std::string(forField); - double expressionValue = performSumAndSubtracting(parser, v); - if (arrayValues_.empty()) { - return {Variant(expressionValue)}; - } else { - arrayValues_.MarkArray(); - return arrayValues_; - } -} - VariantArray ExpressionEvaluator::Evaluate(std::string_view expr, const PayloadValue& v, std::string_view forField) { - arrayValues_.clear(); + arrayValues_.clear(); tokenizer parser(expr); - return Evaluate(parser, v, forField); + forField_ = forField; + state_ = None; + const double expressionValue = performSumAndSubtracting(parser, v); + return (state_ == StateArrayConcat) ? std::move(arrayValues_).MarkArray() : VariantArray{Variant(expressionValue)}; } } // namespace reindexer diff --git a/cpp_src/core/query/expressionevaluator.h b/cpp_src/core/query/expressionevaluator.h index bd32a4353..865219f8e 100644 --- a/cpp_src/core/query/expressionevaluator.h +++ b/cpp_src/core/query/expressionevaluator.h @@ -1,5 +1,6 @@ #pragma once +#include #include "core/keyvalue/variant.h" namespace reindexer { @@ -11,25 +12,34 @@ class TagsMatcher; class ExpressionEvaluator { public: - ExpressionEvaluator(const PayloadType& type, TagsMatcher& tagsMatcher, FunctionExecutor& func); + ExpressionEvaluator(const PayloadType& type, TagsMatcher& tagsMatcher, FunctionExecutor& func) noexcept + : type_(type), tagsMatcher_(tagsMatcher), functionExecutor_(func) {} - VariantArray Evaluate(tokenizer& parser, const PayloadValue& v, std::string_view forField); VariantArray Evaluate(std::string_view expr, const PayloadValue& v, std::string_view forField); private: - double getPrimaryToken(tokenizer& parser, const PayloadValue& v); - double performSumAndSubtracting(tokenizer& parser, const PayloadValue& v); - double performMultiplicationAndDivision(tokenizer& parser, const PayloadValue& v, token& lastTok); - double performArrayConcatenation(tokenizer& parser, const PayloadValue& v, token& lastTok); + struct PrimaryToken { + enum class Type { Scalar, Array, Null }; + + std::optional value; + Type type; + }; + + [[nodiscard]] PrimaryToken getPrimaryToken(tokenizer& parser, const PayloadValue& v, token& outTok); + [[nodiscard]] PrimaryToken handleTokenName(tokenizer& parser, const PayloadValue& v, token& outTok); + [[nodiscard]] double performSumAndSubtracting(tokenizer& parser, const PayloadValue& v); + [[nodiscard]] double performMultiplicationAndDivision(tokenizer& parser, const PayloadValue& v, token& lastTok); + [[nodiscard]] double performArrayConcatenation(tokenizer& parser, const PayloadValue& v, token& lastTok); void captureArrayContent(tokenizer& parser); + [[noreturn]] void throwUnexpectedTokenError(tokenizer& parser, const token& outTok); enum State { None = 0, StateArrayConcat, StateMultiplyAndDivide, StateSumAndSubtract }; const PayloadType& type_; TagsMatcher& tagsMatcher_; FunctionExecutor& functionExecutor_; - std::string forField_; + std::string_view forField_; VariantArray arrayValues_; State state_ = None; }; diff --git a/cpp_src/core/query/query.cc b/cpp_src/core/query/query.cc index d17801e3b..2e7b1da3e 100644 --- a/cpp_src/core/query/query.cc +++ b/cpp_src/core/query/query.cc @@ -236,8 +236,7 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { hasExpressions = ser.GetVarUint(); val.emplace_back(ser.GetVariant().EnsureHold()); } - if (isArray) val.MarkArray(); - Set(std::move(field), std::move(val), hasExpressions); + Set(std::move(field), std::move(val.MarkArray(isArray)), hasExpressions); break; } case QueryUpdateField: { @@ -250,8 +249,7 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { hasExpressions = ser.GetVarUint(); val.emplace_back(ser.GetVariant().EnsureHold()); } - if (isArray) val.MarkArray(); - Set(std::move(field), std::move(val), hasExpressions); + Set(std::move(field), std::move(val.MarkArray(isArray)), hasExpressions); break; } case QueryUpdateObject: { @@ -259,7 +257,7 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { std::string field(ser.GetVString()); bool hasExpressions = false; int numValues = ser.GetVarUint(); - if (ser.GetVarUint() == 1) val.MarkArray(); + val.MarkArray(ser.GetVarUint() == 1); while (numValues--) { hasExpressions = ser.GetVarUint(); val.emplace_back(ser.GetVariant().EnsureHold()); diff --git a/cpp_src/core/query/query.h b/cpp_src/core/query/query.h index 6fc22d3f8..00b2b7203 100644 --- a/cpp_src/core/query/query.h +++ b/cpp_src/core/query/query.h @@ -281,9 +281,8 @@ class Query { Query &Set(std::string field, const std::vector &l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); - value.MarkArray(); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(*it); - return Set(std::move(field), std::move(value), hasExpressions); + return Set(std::move(field), std::move(value.MarkArray()), hasExpressions); } template Query &&Set(std::string field, const std::vector &l, bool hasExpressions = false) && { @@ -335,9 +334,8 @@ class Query { Query &SetObject(std::string field, const std::vector &l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); - value.MarkArray(); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(Variant(*it)); - return SetObject(std::move(field), std::move(value), hasExpressions); + return SetObject(std::move(field), std::move(value.MarkArray()), hasExpressions); } template Query &&SetObject(std::string field, const std::vector &l, bool hasExpressions = false) && { diff --git a/cpp_src/core/query/queryentry.cc b/cpp_src/core/query/queryentry.cc index e242d5b52..a7cc7f4e3 100644 --- a/cpp_src/core/query/queryentry.cc +++ b/cpp_src/core/query/queryentry.cc @@ -1,6 +1,9 @@ #include "queryentry.h" + #include #include +#include "core/nsselecter/joinedselector.h" +#include "core/nsselecter/joinedselectormock.h" #include "core/payload/payloadiface.h" #include "query.h" #include "tools/serializer.h" @@ -9,6 +12,44 @@ namespace reindexer { +template +std::string JoinQueryEntry::Dump(const std::vector &joinedSelectors) const { + WrSerializer ser; + const auto &js = joinedSelectors.at(joinIndex); + const auto &q = js.JoinQuery(); + ser << js.Type() << " (" << q.GetSQL() << ") ON "; + ser << '('; + for (const auto &jqe : q.joinEntries_) { + if (&jqe != &q.joinEntries_.front()) { + ser << ' ' << jqe.op_ << ' '; + } else { + assertrx(jqe.op_ == OpAnd); + } + ser << q._namespace << '.' << jqe.joinIndex_ << ' ' << InvertJoinCondition(jqe.condition_) << ' ' << jqe.index_; + } + ser << ')'; + return std::string{ser.Slice()}; +} +template std::string JoinQueryEntry::Dump(const JoinedSelectors &) const; +template std::string JoinQueryEntry::Dump(const std::vector &) const; + +template +std::string JoinQueryEntry::DumpOnCondition(const std::vector &joinedSelectors) const { + WrSerializer ser; + const auto &js = joinedSelectors.at(joinIndex); + const auto &q = js.JoinQuery(); + ser << js.Type() << " ON ("; + for (const auto &jqe : q.joinEntries_) { + if (&jqe != &q.joinEntries_.front()) { + ser << ' ' << jqe.op_ << ' '; + } + ser << q._namespace << '.' << jqe.joinIndex_ << ' ' << InvertJoinCondition(jqe.condition_) << ' ' << jqe.index_; + } + ser << ')'; + return std::string{ser.Slice()}; +} +template std::string JoinQueryEntry::DumpOnCondition(const JoinedSelectors &) const; + bool QueryEntry::operator==(const QueryEntry &obj) const { return condition == obj.condition && index == obj.index && idxNo == obj.idxNo && distinct == obj.distinct && values.RelaxCompare(obj.values) == 0; @@ -31,6 +72,20 @@ std::string QueryEntry::Dump() const { return std::string{ser.Slice()}; } +std::string QueryEntry::DumpBrief() const { + WrSerializer ser; + { + ser << index << ' ' << condition << ' '; + const bool severalValues = (values.size() > 1); + if (severalValues) { + ser << "(...)"; + } else { + ser << '\'' << values.front().As() << '\''; + } + } + return std::string(ser.Slice()); +} + AggregateEntry::AggregateEntry(AggType type, h_vector fields, SortingEntries sort, unsigned limit, unsigned offset) : type_(type), fields_(std::move(fields)), sortingEntries_{std::move(sort)}, limit_(limit), offset_(offset) { switch (type_) { diff --git a/cpp_src/core/query/queryentry.h b/cpp_src/core/query/queryentry.h index 9391493b1..d2b53ebf8 100644 --- a/cpp_src/core/query/queryentry.h +++ b/cpp_src/core/query/queryentry.h @@ -25,23 +25,10 @@ struct JoinQueryEntry { bool operator!=(const JoinQueryEntry &other) const noexcept { return !operator==(other); } template - std::string Dump(const std::vector &joinedSelectors) const { - WrSerializer ser; - const auto &js = joinedSelectors.at(joinIndex); - const auto &q = js.JoinQuery(); - ser << js.Type() << " (" << q.GetSQL() << ") ON "; - ser << '('; - for (const auto &jqe : q.joinEntries_) { - if (&jqe != &q.joinEntries_.front()) { - ser << ' ' << jqe.op_ << ' '; - } else { - assertrx(jqe.op_ == OpAnd); - } - ser << q._namespace << '.' << jqe.joinIndex_ << ' ' << InvertJoinCondition(jqe.condition_) << ' ' << jqe.index_; - } - ser << ')'; - return std::string{ser.Slice()}; - } + std::string Dump(const std::vector &joinedSelectors) const; + + template + std::string DumpOnCondition(const std::vector &joinedSelectors) const; }; struct QueryEntry { @@ -63,6 +50,7 @@ struct QueryEntry { VariantArray values; std::string Dump() const; + std::string DumpBrief() const; }; class BetweenFieldsQueryEntry { @@ -136,6 +124,8 @@ class QueryEntries static bool checkIfSatisfyCondition(const QueryEntry &, const ConstPayload &, TagsMatcher &); static bool checkIfSatisfyCondition(const BetweenFieldsQueryEntry &, const ConstPayload &, TagsMatcher &); static bool checkIfSatisfyCondition(const VariantArray &lValues, CondType, const VariantArray &rValues); + +protected: static void dumpEqualPositions(size_t level, WrSerializer &ser, const EqualPositions_t &equalPositions) { for (const auto &eq : equalPositions) { for (size_t i = 0; i < level; ++i) { @@ -149,6 +139,7 @@ class QueryEntries ser << ")\n"; } } + template static void dump(size_t level, const_iterator begin, const_iterator end, const std::vector &joinedSelectors, WrSerializer &ser) { for (const_iterator it = begin; it != end; ++it) { @@ -208,11 +199,24 @@ struct QueryJoinEntry { bool operator==(const QueryJoinEntry &) const noexcept; bool operator!=(const QueryJoinEntry &qje) const noexcept { return !operator==(qje); } OpType op_ = OpAnd; - CondType condition_ = CondEq; - std::string index_; - std::string joinIndex_; - int idxNo = -1; - bool reverseNamespacesOrder = false; + CondType condition_ = CondEq; ///< Condition applied to expression: index_ COND joinIndex_ + std::string index_; ///< main ns index field name + std::string joinIndex_; ///< joining ns index field name + int idxNo = -1; ///< index_ field Index number in main ns + bool reverseNamespacesOrder = false; ///< controls SQL encoding order + ///< false: mainNs.index Condition joinNs.joinIndex + ///< true: joinNs.joinIndex Invert(Condition) mainNs.index + + template + std::string DumpCondition(const JS &joinedSelector, bool needOp = false) const { + WrSerializer ser; + const auto &q = joinedSelector.JoinQuery(); + if (needOp) { + ser << ' ' << op_ << ' '; + } + ser << q._namespace << '.' << joinIndex_ << ' ' << InvertJoinCondition(condition_) << ' ' << index_; + return std::string{ser.Slice()}; + } }; struct SortingEntry { diff --git a/cpp_src/core/query/sql/sqlencoder.cc b/cpp_src/core/query/sql/sqlencoder.cc index 4ed0e0bd5..aa82c1110 100644 --- a/cpp_src/core/query/sql/sqlencoder.cc +++ b/cpp_src/core/query/sql/sqlencoder.cc @@ -1,4 +1,5 @@ #include "core/query/sql/sqlencoder.h" + #include "core/keyvalue/geometry.h" #include "core/nsselecter/sortexpression.h" #include "core/queryresults/aggregationresult.h" @@ -7,21 +8,6 @@ namespace reindexer { -const char *SQLEncoder::JoinTypeName(JoinType type) { - switch (type) { - case JoinType::InnerJoin: - return "INNER JOIN"; - case JoinType::OrInnerJoin: - return "OR INNER JOIN"; - case JoinType::LeftJoin: - return "LEFT JOIN"; - case JoinType::Merge: - return "MERGE"; - default: - return ""; - } -} - static void indexToSql(const std::string &index, WrSerializer &ser) { if (index.find('+') == std::string::npos) { ser << index; @@ -68,7 +54,7 @@ SQLEncoder::SQLEncoder(const Query &q) : query_(q) {} void SQLEncoder::DumpSingleJoinQuery(size_t idx, WrSerializer &ser, bool stripArgs) const { assertrx(idx < query_.joinQueries_.size()); const auto &jq = query_.joinQueries_[idx]; - ser << ' ' << JoinTypeName(jq.joinType); + ser << ' ' << jq.joinType; if (jq.entries.Empty() && jq.count == QueryEntry::kDefaultLimit && jq.sortingEntries_.empty()) { ser << ' ' << jq._namespace << " ON "; } else { @@ -101,7 +87,7 @@ void SQLEncoder::dumpJoined(WrSerializer &ser, bool stripArgs) const { void SQLEncoder::dumpMerged(WrSerializer &ser, bool stripArgs) const { for (auto &me : query_.mergeQueries_) { - ser << ' ' << JoinTypeName(me.joinType) << "( "; + ser << ' ' << me.joinType << "( "; me.GetSQL(ser, stripArgs); ser << ')'; } diff --git a/cpp_src/core/query/sql/sqlencoder.h b/cpp_src/core/query/sql/sqlencoder.h index 2e9e1cc5e..c1b217609 100644 --- a/cpp_src/core/query/sql/sqlencoder.h +++ b/cpp_src/core/query/sql/sqlencoder.h @@ -22,11 +22,6 @@ class SQLEncoder { /// @param stripArgs - replace condition values with '?'. void DumpSingleJoinQuery(size_t idx, WrSerializer &ser, bool stripArgs) const; - /// Get readaby Join Type - /// @param type - join tyoe - /// @return string with join type name - static const char *JoinTypeName(JoinType type); - protected: /// Builds print version of a query with join in sql format. /// @param ser - serializer to store SQL string diff --git a/cpp_src/core/query/sql/sqlparser.cc b/cpp_src/core/query/sql/sqlparser.cc index 0fd2162b1..8549d03ae 100644 --- a/cpp_src/core/query/sql/sqlparser.cc +++ b/cpp_src/core/query/sql/sqlparser.cc @@ -6,6 +6,7 @@ #include "core/queryresults/aggregationresult.h" #include "core/type_consts_helpers.h" #include "sqltokentype.h" +#include "vendor/double-conversion/double-conversion.h" #include "vendor/gason/gason.h" namespace reindexer { @@ -23,7 +24,7 @@ bool SQLParser::reachedAutocompleteToken(tokenizer &parser, const token &tok) { } token SQLParser::peekSqlToken(tokenizer &parser, int tokenType, bool toLower) { - token tok = parser.peek_token(toLower); + token tok = parser.peek_token(toLower ? tokenizer::flags::to_lower : tokenizer::flags::no_flags); bool eof = ((parser.getPos() + tok.text().length()) == parser.length()); if (ctx_.autocompleteMode && !tok.text().empty() && reachedAutocompleteToken(parser, tok)) { size_t tokenLen = 0; @@ -91,12 +92,16 @@ int SQLParser::selectParse(tokenizer &parser) { tok = peekSqlToken(parser, SingleSelectFieldSqlToken); if (name.text() == "count"sv) { query_.calcTotal = ModeAccurateTotal; - if (!wasSelectFilter) query_.count = 0; + if (!wasSelectFilter) { + query_.count = 0; + } tok = parser.next_token(); if (tok.text() != "*") throw Error(errParseSQL, "Expected '*', but found '%s' in query, %s", tok.text(), parser.where()); } else if (name.text() == "count_cached"sv) { query_.calcTotal = ModeCachedTotal; - if (!wasSelectFilter) query_.count = 0; + if (!wasSelectFilter) { + query_.count = 0; + } tok = parser.next_token(); if (tok.text() != "*"sv) throw Error(errParseSQL, "Expected '*', but found '%s' in query, %s", tok.text(), parser.where()); } else if (name.text() == "rank"sv) { @@ -146,7 +151,7 @@ int SQLParser::selectParse(tokenizer &parser) { break; } } - query_.aggregations_.push_back(std::move(entry)); + query_.aggregations_.emplace_back(std::move(entry)); } else { throw Error(errParams, "Unknown function name SQL - %s, %s", name.text(), parser.where()); } @@ -162,7 +167,7 @@ int SQLParser::selectParse(tokenizer &parser) { if (!query_.CanAddSelectFilter()) { throw Error(errConflict, kAggregationWithSelectFieldsMsgError); } - query_.selectFilter_.push_back(std::string(nameWithCase.text())); + query_.selectFilter_.emplace_back(nameWithCase.text()); query_.count = QueryEntry::kDefaultLimit; wasSelectFilter = true; } else if (name.text() == "*"sv) { @@ -318,8 +323,14 @@ Variant token2kv(const token &currTok, tokenizer &parser, bool allowComposite) { return detectValueType(currTok).EvaluateOneOf( [&](KeyValueType::Int64) { return Variant(int64_t(stoll(value))); }, [&](KeyValueType::Double) { - char *p = 0; - return Variant(double(strtod(value.data(), &p))); + try { + using double_conversion::StringToDoubleConverter; + static const StringToDoubleConverter converter{StringToDoubleConverter::NO_FLAGS, NAN, NAN, nullptr, nullptr}; + int countOfCharsParsedAsDouble; + return Variant(converter.StringToDouble(value.data(), value.size(), &countOfCharsParsedAsDouble)); + } catch (...) { + throw Error(errParseSQL, "Unable to convert '%s' to double value", value); + } }, [&](KeyValueType::String) { return Variant(make_key_string(value.data(), value.length())); }, [](OneOf 0) { @@ -488,17 +500,17 @@ UpdateEntry SQLParser::parseUpdateField(tokenizer &parser) { size_t startPos = parser.getPos(); bool withArrayExpressions = false; - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); if (tok.text() == "["sv) { updateField.Values().MarkArray(); for (;;) { - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); if (tok.text() == "]") { if (updateField.Values().empty()) break; throw Error(errParseSQL, "Expected field value, but found ']' in query, %s", parser.where()); } addUpdateValue(tok, parser, updateField); - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); if (tok.text() == "]"sv) break; if (tok.text() != ","sv) { throw Error(errParseSQL, "Expected ']' or ',', but found '%s' in query, %s", tok.text(), parser.where()); @@ -748,7 +760,7 @@ void SQLParser::parseEqualPositions(tokenizer &parser, std::vector + S &DumpFields(S &os) { + os << '['; + bool first = true; + for (const auto &f : fields) { + if (!first) { + os << ", "; + } + first = false; + os << f; + } + os << ']'; + return os; + } private: std::optional value_ = std::nullopt; diff --git a/cpp_src/core/queryresults/queryresults.cc b/cpp_src/core/queryresults/queryresults.cc index acea38573..1ba73eaec 100644 --- a/cpp_src/core/queryresults/queryresults.cc +++ b/cpp_src/core/queryresults/queryresults.cc @@ -12,12 +12,11 @@ namespace reindexer { -void QueryResults::AddNamespace(std::shared_ptr ns, const NsContext &ctx) { - assertrx(ctx.noLock); +void QueryResults::AddNamespace(NamespaceImplPtr ns, [[maybe_unused]] bool noLock) { + assertrx(noLock); const NamespaceImpl *nsPtr = ns.get(); - auto strHolder = ns->StrHolder(ctx); - const auto it = - std::find_if(nsData_.cbegin(), nsData_.cend(), [nsPtr](const NsDataHolder &nsData) { return nsData.ns.get() == nsPtr; }); + auto strHolder = ns->strHolder(); + const auto it = std::find_if(nsData_.cbegin(), nsData_.cend(), [nsPtr](const NsDataHolder &nsData) { return nsData.ns == nsPtr; }); if (it != nsData_.cend()) { assertrx(it->strHolder.get() == strHolder.get()); return; @@ -25,8 +24,19 @@ void QueryResults::AddNamespace(std::shared_ptr ns, const NsConte nsData_.emplace_back(std::move(ns), std::move(strHolder)); } +void QueryResults::AddNamespace(NamespaceImpl *ns, [[maybe_unused]] bool noLock) { + assertrx(noLock); + auto strHolder = ns->strHolder(); + const auto it = std::find_if(nsData_.cbegin(), nsData_.cend(), [ns](const NsDataHolder &nsData) { return nsData.ns == ns; }); + if (it != nsData_.cend()) { + assertrx(it->strHolder.get() == strHolder.get()); + return; + } + nsData_.emplace_back(ns, std::move(strHolder)); +} + void QueryResults::RemoveNamespace(const NamespaceImpl *ns) { - const auto it = std::find_if(nsData_.begin(), nsData_.end(), [ns](const NsDataHolder &nsData) { return nsData.ns.get() == ns; }); + const auto it = std::find_if(nsData_.begin(), nsData_.end(), [ns](const NsDataHolder &nsData) { return nsData.ns == ns; }); assertrx(it != nsData_.end()); nsData_.erase(it); } @@ -447,38 +457,19 @@ void QueryResults::AddItem(Item &item, bool withData, bool enableHold) { } } -const TagsMatcher &QueryResults::getTagsMatcher(int nsid) const { - assertrx(nsid < int(ctxs.size())); - return ctxs[nsid].tagsMatcher_; -} +const TagsMatcher &QueryResults::getTagsMatcher(int nsid) const { return ctxs[nsid].tagsMatcher_; } -const PayloadType &QueryResults::getPayloadType(int nsid) const { - assertrx(nsid < int(ctxs.size())); - return ctxs[nsid].type_; -} +const PayloadType &QueryResults::getPayloadType(int nsid) const { return ctxs[nsid].type_; } -const FieldsSet &QueryResults::getFieldsFilter(int nsid) const { - assertrx(nsid < int(ctxs.size())); - return ctxs[nsid].fieldsFilter_; -} +const FieldsSet &QueryResults::getFieldsFilter(int nsid) const { return ctxs[nsid].fieldsFilter_; } -TagsMatcher &QueryResults::getTagsMatcher(int nsid) { - assertrx(nsid < int(ctxs.size())); - return ctxs[nsid].tagsMatcher_; -} +TagsMatcher &QueryResults::getTagsMatcher(int nsid) { return ctxs[nsid].tagsMatcher_; } -PayloadType &QueryResults::getPayloadType(int nsid) { - assertrx(nsid < int(ctxs.size())); - return ctxs[nsid].type_; -} +PayloadType &QueryResults::getPayloadType(int nsid) { return ctxs[nsid].type_; } -std::shared_ptr QueryResults::getSchema(int nsid) const { - assertrx(nsid < int(ctxs.size())); - return ctxs[nsid].schema_; -} +std::shared_ptr QueryResults::getSchema(int nsid) const { return ctxs[nsid].schema_; } int QueryResults::getNsNumber(int nsid) const { - assertrx(nsid < int(ctxs.size())); assertrx(ctxs[nsid].schema_); return ctxs[nsid].schema_->GetProtobufNsNumber(); } @@ -492,4 +483,10 @@ void QueryResults::addNSContext(const PayloadType &type, const TagsMatcher &tags ctxs.push_back(Context(type, tagsMatcher, filter, std::move(schema))); } +QueryResults::NsDataHolder::NsDataHolder(QueryResults::NamespaceImplPtr &&_ns, StringsHolderPtr &&strHldr) noexcept + : nsPtr_{std::move(_ns)}, ns(nsPtr_.get()), strHolder{std::move(strHldr)} {} + +QueryResults::NsDataHolder::NsDataHolder(NamespaceImpl *_ns, StringsHolderPtr &&strHldr) noexcept + : ns(_ns), strHolder(std::move(strHldr)) {} + } // namespace reindexer diff --git a/cpp_src/core/queryresults/queryresults.h b/cpp_src/core/queryresults/queryresults.h index 74b3ffe9a..e3e4d11f1 100644 --- a/cpp_src/core/queryresults/queryresults.h +++ b/cpp_src/core/queryresults/queryresults.h @@ -35,6 +35,8 @@ class ItemIterator; class QueryResults { public: + using NamespaceImplPtr = intrusive_ptr; + QueryResults(int flags = 0); QueryResults(const ItemRefVector::const_iterator &b, const ItemRefVector::const_iterator &e); QueryResults(std::initializer_list l); @@ -122,10 +124,15 @@ class QueryResults { ItemRefVector &Items() { return items_; } const ItemRefVector &Items() const { return items_; } int GetJoinedNsCtxIndex(int nsid) const; - void AddNamespace(std::shared_ptr, const NsContext &); + // Add owning ns pointer + // noLock has always to be 'true' (i.e. this method can only be called unders Namespace's lock) + void AddNamespace(NamespaceImplPtr, bool noLock); + // Add non-owning ns pointer + // noLock has always to be 'true' (i.e. this method can only be called unders Namespace's lock) + void AddNamespace(NamespaceImpl *, bool noLock); void RemoveNamespace(const NamespaceImpl *ns); bool IsNamespaceAdded(const NamespaceImpl *ns) const noexcept { - return std::find_if(nsData_.cbegin(), nsData_.cend(), [ns](const NsDataHolder &nsData) { return nsData.ns.get() == ns; }) != + return std::find_if(nsData_.cbegin(), nsData_.cend(), [ns](const NsDataHolder &nsData) { return nsData.ns == ns; }) != nsData_.cend(); } void MarkAsWALQuery() noexcept { isWalQuery_ = true; } @@ -143,15 +150,20 @@ class QueryResults { std::optional activityCtx_; friend InternalRdxContext; friend SelectFunctionsHolder; - struct NsDataHolder { - NsDataHolder(std::shared_ptr &&ns_, StringsHolderPtr &&strHldr) noexcept - : ns{std::move(ns_)}, strHolder{std::move(strHldr)} {} + class NsDataHolder { + public: + NsDataHolder(NamespaceImplPtr &&_ns, StringsHolderPtr &&strHldr) noexcept; + NsDataHolder(NamespaceImpl *_ns, StringsHolderPtr &&strHldr) noexcept; NsDataHolder(const NsDataHolder &) = delete; NsDataHolder(NsDataHolder &&) noexcept = default; NsDataHolder &operator=(const NsDataHolder &) = delete; NsDataHolder &operator=(NsDataHolder &&) = default; - std::shared_ptr ns; + private: + NamespaceImplPtr nsPtr_; + + public: + NamespaceImpl *ns; StringsHolderPtr strHolder; }; diff --git a/cpp_src/core/reindexerimpl.cc b/cpp_src/core/reindexerimpl.cc index f26086305..b34c8778b 100644 --- a/cpp_src/core/reindexerimpl.cc +++ b/cpp_src/core/reindexerimpl.cc @@ -1,4 +1,5 @@ #include "core/reindexerimpl.h" + #include #include #include @@ -344,7 +345,7 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r return Error(errParams, "Namespace name contains invalid character. Only alphas, digits,'_','-', are allowed"); } bool readyToLoadStorage = (nsDef.storage.IsEnabled() && !storagePath_.empty()); - ns = std::make_shared(nsDef.name, observers_); + ns = std::make_shared(nsDef.name, observers_, bgDeleter_); if (nsDef.isTemporary) { ns->awaitMainNs(rdxCtx)->setTemporary(); } @@ -385,7 +386,7 @@ Error ReindexerImpl::openNamespace(std::string_view name, const StorageOpts& sto return Error(errParams, "Namespace name contains invalid character. Only alphas, digits,'_','-', are allowed"); } std::string nameStr(name); - auto ns = std::make_shared(nameStr, observers_); + auto ns = std::make_shared(nameStr, observers_, bgDeleter_); if (storageOpts.IsSlaveMode()) ns->setSlaveMode(rdxCtx); if (storageOpts.IsEnabled() && !storagePath_.empty()) { auto opts = storageOpts; @@ -422,6 +423,7 @@ Error ReindexerImpl::CloseNamespace(std::string_view nsName, const InternalRdxCo Error ReindexerImpl::closeNamespace(std::string_view nsName, const RdxContext& ctx, bool dropStorage, bool enableDropSlave) { Namespace::Ptr ns; + Error err; try { ULock lock(mtx_, &ctx); auto nsIt = namespaces_.find(nsName); @@ -451,10 +453,13 @@ Error ReindexerImpl::closeNamespace(std::string_view nsName, const RdxContext& c } } - } catch (const Error& err) { - return err; + } catch (const Error& e) { + err = e; } - return errOK; + if (ns) { + bgDeleter_.Add(ns->atomicLoadMainNs()); + } + return err; } Error ReindexerImpl::syncDownstream(std::string_view nsName, bool force, const InternalRdxContext& ctx) { @@ -614,13 +619,13 @@ Error ReindexerImpl::applyNsFunction(std::string_view nsName, const InternalRdxC return err; } -#define APPLY_NS_FUNCTION1(needUpdateSys, memFn, arg) \ - return applyNsFunction( \ +#define APPLY_NS_FUNCTION1(needUpdateSys, memFn, arg) \ + return applyNsFunction( \ nsName, ctx, makeCtxStr, arg) -#define APPLY_NS_FUNCTION2(needUpdateSys, memFn, arg1, arg2) \ - return applyNsFunction(nsName, ctx, makeCtxStr, arg1, arg2) +#define APPLY_NS_FUNCTION2(needUpdateSys, memFn, arg1, arg2) \ + return applyNsFunction(nsName, ctx, makeCtxStr, arg1, arg2) Error ReindexerImpl::Insert(std::string_view nsName, Item& item, const InternalRdxContext& ctx) { const auto makeCtxStr = [nsName](WrSerializer& ser) -> WrSerializer& { return ser << "INSERT INTO " << nsName; }; @@ -674,10 +679,6 @@ Error ReindexerImpl::Update(const Query& q, QueryResults& result, const Internal WrSerializer ser; const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? q.GetSQL(ser).Slice() : ""sv, activities_, result); auto ns = getNamespace(q._namespace, rdxCtx); - - QueryStatCalculator statCalculator(long_actions::Logger{ - q, isSystemNamespaceNameFast(q._namespace) ? LongQueriesLoggingParams{} : configProvider_.GetUpdDelLoggingParams()}); - ns->Update(q, result, rdxCtx); if (ns->IsSystem(rdxCtx)) { const std::string kNsName = ns->GetName(rdxCtx); @@ -788,9 +789,6 @@ Error ReindexerImpl::Delete(std::string_view nsName, Item& item, QueryResults& q Error ReindexerImpl::Delete(const Query& q, QueryResults& result, const InternalRdxContext& ctx) { const auto makeCtxStr = [&q](WrSerializer& ser) -> WrSerializer& { return q.GetSQL(ser); }; const std::string_view nsName = q._namespace; - QueryStatCalculator statCalculator(long_actions::Logger{ - q, isSystemNamespaceNameFast(nsName) ? LongQueriesLoggingParams{} : configProvider_.GetUpdDelLoggingParams()}); - APPLY_NS_FUNCTION2(false, Delete, q, result); } @@ -868,7 +866,8 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal QueryStatCalculator statCalculator( std::move(hitter), std::chrono::microseconds(queriesThresholdUS), queriesPerfStatsEnabled || configProvider_.GetSelectLoggingParams().thresholdUs >= 0, - long_actions::Logger{q, isSystemNsRequest ? LongQueriesLoggingParams{} : configProvider_.GetSelectLoggingParams()}); + long_actions::MakeLogger( + q, isSystemNsRequest ? LongQueriesLoggingParams{} : configProvider_.GetSelectLoggingParams())); StatsLocker::StatsLockT statsSelectLck; if (isSystemNsRequest) { @@ -980,20 +979,6 @@ bool ReindexerImpl::isPreResultValuesModeOptimizationAvailable(const Query& jIte return true; } -void ReindexerImpl::prepareJoinResults(const Query& q, QueryResults& result) { - bool thereAreJoins = !q.joinQueries_.empty(); - if (!thereAreJoins) { - for (const Query& mq : q.mergeQueries_) { - if (!mq.joinQueries_.empty()) { - thereAreJoins = true; - break; - } - } - } - if (thereAreJoins) { - result.joined_.resize(1 + q.mergeQueries_.size()); - } -} template JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResults& result, NsLocker& locks, SelectFunctionsHolder& func, std::vector& queryResultsContexts, const RdxContext& rdxCtx) { @@ -1005,9 +990,15 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul // For each joined queries uint32_t joinedSelectorsCount = uint32_t(q.joinQueries_.size()); for (auto& jq : q.joinQueries_) { - if (rx_unlikely(isSystemNamespaceNameFast(jq._namespace))) { + if rx_unlikely (isSystemNamespaceNameFast(jq._namespace)) { throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside JOIN statement", jq._namespace); } + if (rx_unlikely(!jq.joinQueries_.empty())) { + throw Error(errParams, "JOINs nested into the other JOINs are not supported"); + } + if (rx_unlikely(!jq.mergeQueries_.empty())) { + throw Error(errParams, "MERGEs nested into the JOINs are not supported"); + } // Get common results from joined namespaces_ auto jns = locks.Get(jq._namespace); @@ -1065,7 +1056,7 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul queryResultsContexts.emplace_back(jns->payloadType_, jns->tagsMatcher_, FieldsSet(jns->tagsMatcher_, jq.selectFilter_), jns->schema_); - result.AddNamespace(jns, {rdxCtx, true}); + result.AddNamespace(jns, true); if (preResult->dataMode == JoinPreResult::ModeValues) { jItemQ.entries.ExecuteAppropriateForEach( Skip{}, @@ -1097,62 +1088,107 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul return joinedSelectors; } -template +template void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& locks, SelectFunctionsHolder& func, const RdxContext& ctx, - QueryStatCalculator& queryStatCalculator) { + QueryStatCalculator& queryStatCalculator) { auto ns = locks.Get(q._namespace); - assertrx(ns); - if (!ns) { - throw Error(errParams, "Namespace '%s' is not exists", q._namespace); + if rx_unlikely (!ns) { + throw Error(errParams, "Namespace '%s' does not exist", q._namespace); } std::vector joinQueryResultsContexts; - // should be destroyed after results.lockResults() - JoinedSelectors mainJoinedSelectors = prepareJoinedSelectors(q, result, locks, func, joinQueryResultsContexts, ctx); - prepareJoinResults(q, result); + bool thereAreJoins = !q.joinQueries_.empty(); + if (!thereAreJoins) { + for (const Query& mq : q.mergeQueries_) { + if (!mq.joinQueries_.empty()) { + thereAreJoins = true; + break; + } + } + } + + JoinedSelectors mainJoinedSelectors; + ExplainCalc::Duration preselectTimeTotal{0}; + if (thereAreJoins) { + const auto preselectStartTime = ExplainCalc::Clock::now(); + mainJoinedSelectors = prepareJoinedSelectors(q, result, locks, func, joinQueryResultsContexts, ctx); + result.joined_.resize(1 + q.mergeQueries_.size()); + preselectTimeTotal = ExplainCalc::Clock::now() - preselectStartTime; + } IsFTQuery isFtQuery{IsFTQuery::NotSet}; { SelectCtx selCtx(q, nullptr); selCtx.joinedSelectors = mainJoinedSelectors.size() ? &mainJoinedSelectors : nullptr; + selCtx.preResultTimeTotal = preselectTimeTotal; selCtx.contextCollectingMode = true; selCtx.functions = &func; selCtx.nsid = 0; if (!q.mergeQueries_.empty()) { selCtx.isMergeQuery = IsMergeQuery::Yes; - if (!q.sortingEntries_.empty()) { + if rx_unlikely (!q.sortingEntries_.empty()) { throw Error{errNotValid, "Sorting in merge query is not implemented yet"}; // TODO #1449 } - if (!q.aggregations_.empty()) { - throw Error{errNotValid, "Aggregations in merge query is not implemented yet"}; // TODO #1448 + for (const auto& a : q.aggregations_) { + switch (a.Type()) { + case AggCount: + case AggCountCached: + case AggSum: + case AggMin: + case AggMax: + continue; + case AggAvg: + case AggFacet: + case AggDistinct: + case AggUnknown: + throw Error{errNotValid, "Aggregation '%s' in merge query is not implemented yet", + AggTypeToStr(a.Type())}; // TODO #1506 + } } } selCtx.requiresCrashTracking = true; ns->Select(result, selCtx, ctx); - result.AddNamespace(ns, {ctx, true}); + result.AddNamespace(ns, true); isFtQuery = selCtx.isFtQuery; if (selCtx.explain.IsEnabled()) { queryStatCalculator.AddExplain(selCtx.explain); } } - // should be destroyed after results.lockResults() std::vector mergeJoinedSelectors; if (!q.mergeQueries_.empty()) { mergeJoinedSelectors.reserve(q.mergeQueries_.size()); uint8_t counter = 0; + auto hasUnsupportedAggreagations = [](const std::vector& aggVector, AggType& t) -> bool { + for (const auto& a : aggVector) { + if (a.Type() != AggCount || a.Type() != AggCountCached) { + t = a.Type(); + return true; + } + } + t = AggUnknown; + return false; + }; + AggType errType; + if (rx_unlikely((q.HasLimit() || q.HasOffset()) && hasUnsupportedAggreagations(q.aggregations_, errType))) { + throw Error(errParams, "Limit and offset are not supported for aggregations '%s'", AggTypeToStr(errType)); + } for (auto& mq : q.mergeQueries_) { - if (rx_unlikely(isSystemNamespaceNameFast(mq._namespace))) { + if rx_unlikely (isSystemNamespaceNameFast(mq._namespace)) { throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside MERGE statement", mq._namespace); } - if (rx_unlikely(!mq.sortingEntries_.empty())) { + if rx_unlikely (!mq.sortingEntries_.empty()) { throw Error(errParams, "Sorting in inner merge query is not allowed"); } - if (rx_unlikely(!mq.aggregations_.empty())) { + if rx_unlikely (!mq.aggregations_.empty() || mq.calcTotal != ModeNoTotal) { throw Error(errParams, "Aggregations in inner merge query is not allowed"); } - if (rx_unlikely(mq.HasLimit() || mq.HasOffset())) { + if rx_unlikely (mq.HasLimit() || mq.HasOffset()) { throw Error(errParams, "Limit and offset in inner merge query is not allowed"); } + if rx_unlikely (!mq.mergeQueries_.empty()) { + throw Error(errParams, "MERGEs nested into the MERGEs are not supported"); + } + auto mns = locks.Get(mq._namespace); assertrx(mns); SelectCtx mctx(mq, &q); @@ -1164,15 +1200,10 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& mergeJoinedSelectors.emplace_back(prepareJoinedSelectors(mq, result, locks, func, joinQueryResultsContexts, ctx)); mctx.joinedSelectors = mergeJoinedSelectors.back().size() ? &mergeJoinedSelectors.back() : nullptr; mctx.requiresCrashTracking = true; - mns->Select(result, mctx, ctx); - result.AddNamespace(mns, {ctx, true}); + result.AddNamespace(mns, true); } - ItemRefVector& itemRefVec = result.Items(); - if (q.calcTotal) { - result.totalCount = itemRefVec.size(); - } if (static_cast(q.start) >= itemRefVec.size()) { result.Erase(itemRefVec.begin(), itemRefVec.end()); return; @@ -1189,9 +1220,6 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& for (const auto& jctx : joinQueryResultsContexts) result.addNSContext(jctx.type_, jctx.tagsMatcher_, jctx.fieldsFilter_, jctx.schema_); } -template void ReindexerImpl::doSelect(const Query&, QueryResults&, NsLocker&, SelectFunctionsHolder&, const RdxContext&, - QueryStatCalculator&); - Error ReindexerImpl::Commit(std::string_view /*_namespace*/) { try { // getNamespace(_namespace)->FlushStorage(); @@ -1324,6 +1352,7 @@ Error ReindexerImpl::EnumNamespaces(std::vector& defs, EnumNamespa void ReindexerImpl::backgroundRoutine(net::ev::dynamic_loop& loop) { static const RdxContext dummyCtx; auto nsBackground = [&]() { + bgDeleter_.DeleteUnique(); auto nsarray = getNamespacesNames(dummyCtx); for (const auto& name : nsarray) { try { @@ -1692,7 +1721,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) } } resultLock = statsLocker_.LockIfRequired(sysNsName, ctx); - sysNs->Refill(items, NsContext(ctx)); + sysNs->Refill(items, ctx); }; if (sysNsName == kPerfStatsNamespace) { @@ -1740,7 +1769,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) auto err = item.FromJSON(ser.Slice()); if (!err.ok()) throw err; } - queriesperfstatsNs->Refill(items, NsContext(ctx)); + queriesperfstatsNs->Refill(items, ctx); } } else if (sysNsName == kActivityStatsNamespace) { const auto data = activities_.List(); @@ -1758,7 +1787,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) auto err = item.FromJSON(ser.Slice()); if (!err.ok()) throw err; } - activityNs->Refill(items, NsContext(ctx)); + activityNs->Refill(items, ctx); } else if (sysNsName == kClientsStatsNamespace) { if (clientsStats_) { std::vector clientInf; @@ -1789,7 +1818,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) auto err = item.FromJSON(ser.Slice()); if (!err.ok()) throw err; } - clientsNs->Refill(items, NsContext(ctx)); + clientsNs->Refill(items, ctx); } } return resultLock; diff --git a/cpp_src/core/reindexerimpl.h b/cpp_src/core/reindexerimpl.h index b3d046048..f8ef5736d 100644 --- a/cpp_src/core/reindexerimpl.h +++ b/cpp_src/core/reindexerimpl.h @@ -200,14 +200,13 @@ class ReindexerImpl { std::unordered_map mtxMap_; }; - template + template void doSelect(const Query &q, QueryResults &result, NsLocker &locks, SelectFunctionsHolder &func, const RdxContext &ctx, - QueryStatCalculator &queryStatCalculator); + QueryStatCalculator &queryStatCalculator); struct QueryResultsContext; template JoinedSelectors prepareJoinedSelectors(const Query &q, QueryResults &result, NsLocker &locks, SelectFunctionsHolder &func, std::vector &, const RdxContext &ctx); - void prepareJoinResults(const Query &q, QueryResults &result); static bool isPreResultValuesModeOptimizationAvailable(const Query &jItemQ, const NamespaceImpl::Ptr &jns, const Query &mainQ); FilterNsNamesT detectFilterNsNames(const Query &q); @@ -247,6 +246,7 @@ class ReindexerImpl { BackgroundThread backgroundThread_; BackgroundThread storageFlushingThread_; std::atomic dbDestroyed_ = {false}; + BackgroundNamespaceDeleter bgDeleter_; QueriesStatTracer queriesStatTracker_; UpdatesObservers observers_; diff --git a/cpp_src/core/selectfunc/functionexecutor.cc b/cpp_src/core/selectfunc/functionexecutor.cc index 8f42c7ac1..f384bd3ac 100644 --- a/cpp_src/core/selectfunc/functionexecutor.cc +++ b/cpp_src/core/selectfunc/functionexecutor.cc @@ -5,20 +5,17 @@ namespace reindexer { -FunctionExecutor::FunctionExecutor(NamespaceImpl& ns) noexcept : ns_(ns) {} - Variant FunctionExecutor::Execute(SelectFuncStruct& funcData) { if (funcData.funcName == "now") { - std::string mode = "sec"; + std::string_view mode = "sec"; if (!funcData.funcArgs.empty() && !funcData.funcArgs.front().empty()) { mode = funcData.funcArgs.front(); } return Variant(getTimeNow(mode)); } else if (funcData.funcName == "serial") { return Variant(ns_.GetSerial(funcData.field)); - } else { - throw Error(errParams, "Unknown function %s", funcData.field); } + throw Error(errParams, "Unknown function '%s'. Field: '%s'", funcData.funcName, funcData.field); } } // namespace reindexer diff --git a/cpp_src/core/selectfunc/functionexecutor.h b/cpp_src/core/selectfunc/functionexecutor.h index 471653147..1634805af 100644 --- a/cpp_src/core/selectfunc/functionexecutor.h +++ b/cpp_src/core/selectfunc/functionexecutor.h @@ -9,7 +9,7 @@ struct SelectFuncStruct; class FunctionExecutor { public: - explicit FunctionExecutor(NamespaceImpl& ns) noexcept; + explicit FunctionExecutor(NamespaceImpl& ns) noexcept : ns_(ns) {} Variant Execute(SelectFuncStruct& funcData); private: diff --git a/cpp_src/core/selectfunc/selectfuncparser.cc b/cpp_src/core/selectfunc/selectfuncparser.cc index a0172c1b5..1836f7441 100644 --- a/cpp_src/core/selectfunc/selectfuncparser.cc +++ b/cpp_src/core/selectfunc/selectfuncparser.cc @@ -10,21 +10,22 @@ namespace reindexer { SelectFuncStruct &SelectFuncParser::Parse(const std::string &query) { tokenizer parser(query); - token tok = parser.next_token(false); + token tok = parser.next_token(tokenizer::flags::no_flags); selectFuncStruct_.field = std::string(tok.text()); auto dotPos = tok.text().find('.'); if (dotPos == std::string_view::npos) { - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); if (tok.text() != "=") { throw Error(errParams, "`=` is expected, but found `%s`", tok.text()); } - ParseFunction(parser, false); + token ftok; + ParseFunction(parser, false, ftok); } else { token ftok(TokenName); ftok.text_.assign(tok.text_.begin() + dotPos + 1, tok.text_.end()); - ParseFunction(parser, false, std::move(ftok)); + ParseFunction(parser, false, ftok); } if (!selectFuncStruct_.isFunction) { @@ -38,7 +39,7 @@ SelectFuncStruct &SelectFuncParser::Parse(const std::string &query) { void SelectFuncParser::parsePositionalAndNamedArgs(tokenizer &parser, const Args &args) { using namespace std::string_view_literals; token tok; - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); if (!(tok.type == TokenSymbol && tok.text() == "("sv)) { throw Error(errParseDSL, "%s: An open parenthesis is required, but found `%s`", selectFuncStruct_.funcName, tok.text()); } @@ -48,11 +49,11 @@ void SelectFuncParser::parsePositionalAndNamedArgs(tokenizer &parser, const Args NamedArgState expectedToken = NamedArgState::Name; while (!parser.end()) { - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); switch (tok.type) { case TokenSymbol: if (tok.text() == ")"sv) { - token nextTok = parser.next_token(false); + token nextTok = parser.next_token(tokenizer::flags::no_flags); if (nextTok.text().length() > 0) { throw Error(errParseDSL, "%s: Unexpected character `%s` after closing parenthesis.", selectFuncStruct_.funcName, nextTok.text()); @@ -170,10 +171,10 @@ void SelectFuncParser::parsePositionalAndNamedArgs(tokenizer &parser, const Args } } -SelectFuncStruct &SelectFuncParser::ParseFunction(tokenizer &parser, bool partOfExpression, token tok) { +SelectFuncStruct &SelectFuncParser::ParseFunction(tokenizer &parser, bool partOfExpression, token &tok) { using namespace std::string_view_literals; if (tok.text().empty()) { - tok = parser.next_token(true); + tok = parser.next_token(); } if (tok.text() == "snippet") { selectFuncStruct_.func = Snippet(); @@ -188,14 +189,14 @@ SelectFuncStruct &SelectFuncParser::ParseFunction(tokenizer &parser, bool partOf } selectFuncStruct_.funcName = std::string(tok.text()); - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); if (tok.text() == "(") { std::string arg; while (!parser.end()) { - tok = parser.next_token(false); + tok = parser.next_token(tokenizer::flags::no_flags); if (tok.text() == ")") { if (!partOfExpression) { - token nextTok = parser.next_token(false); + token nextTok = parser.next_token(tokenizer::flags::no_flags); if (nextTok.text().length() > 0) { throw Error(errParseDSL, "%s: Unexpected character `%s` after closing parenthesis", selectFuncStruct_.funcName, nextTok.text()); @@ -223,7 +224,7 @@ SelectFuncStruct &SelectFuncParser::ParseFunction(tokenizer &parser, bool partOf return selectFuncStruct_; } -bool SelectFuncParser::IsFunction(std::string_view val) { +bool SelectFuncParser::IsFunction(std::string_view val) noexcept { if (val.length() < 3) return false; size_t i = 0; @@ -257,7 +258,7 @@ bool SelectFuncParser::IsFunction(std::string_view val) { return false; } -bool SelectFuncParser::IsFunction(const VariantArray &val) { +bool SelectFuncParser::IsFunction(const VariantArray &val) noexcept { if (val.size() != 1) return false; if (!val.front().Type().Is()) return false; return IsFunction(static_cast(val.front())); diff --git a/cpp_src/core/selectfunc/selectfuncparser.h b/cpp_src/core/selectfunc/selectfuncparser.h index 8c0455df7..dba14b179 100644 --- a/cpp_src/core/selectfunc/selectfuncparser.h +++ b/cpp_src/core/selectfunc/selectfuncparser.h @@ -16,7 +16,7 @@ class BaseFunctionCtx; class FuncNone { public: - bool Process(ItemRef &, PayloadType &, const SelectFuncStruct &, std::vector &) { return false; } + bool Process(ItemRef &, PayloadType &, const SelectFuncStruct &, std::vector &) noexcept { return false; } }; template @@ -53,13 +53,10 @@ struct SelectFuncStruct { class SelectFuncParser { public: - SelectFuncParser() {} - ~SelectFuncParser() {} - SelectFuncStruct &Parse(const std::string &query); - SelectFuncStruct &ParseFunction(tokenizer &parser, bool partOfExpression, token tok = token()); - static bool IsFunction(std::string_view val); - static bool IsFunction(const VariantArray &val); + SelectFuncStruct &ParseFunction(tokenizer &parser, bool partOfExpression, token &tok); + static bool IsFunction(std::string_view val) noexcept; + static bool IsFunction(const VariantArray &val) noexcept; protected: struct Args { diff --git a/cpp_src/core/type_consts.h b/cpp_src/core/type_consts.h index 6b2b50956..131d8a2db 100644 --- a/cpp_src/core/type_consts.h +++ b/cpp_src/core/type_consts.h @@ -36,6 +36,7 @@ typedef enum IndexType { IndexTtl = 18, IndexRTree = 19, IndexUuidHash = 20, + IndexUuidStore = 21, } IndexType; typedef enum QueryItemType { diff --git a/cpp_src/core/type_consts_helpers.cc b/cpp_src/core/type_consts_helpers.cc index 323ddc829..18fd15e1c 100644 --- a/cpp_src/core/type_consts_helpers.cc +++ b/cpp_src/core/type_consts_helpers.cc @@ -1,4 +1,5 @@ #include "type_consts_helpers.h" + #include "tools/errors.h" namespace reindexer { @@ -23,9 +24,9 @@ namespace reindexer { case CondEmpty: case CondLike: case CondDWithin: - default: - throw Error(errForbidden, "Not invertible conditional operator '%s(%d)' in query", CondTypeToStr(cond), cond); + break; } + throw Error(errForbidden, "Not invertible conditional operator '%s(%d)' in query", CondTypeToStr(cond), cond); } [[nodiscard]] std::string_view CondTypeToStr(CondType t) { @@ -55,9 +56,8 @@ namespace reindexer { return "CondLike"sv; case CondDWithin: return "CondDWithin"sv; - default: - throw Error{errNotValid, "Invalid condition type: %d", t}; } + throw Error{errNotValid, "Invalid condition type: %d", t}; } [[nodiscard]] std::string_view TagTypeToStr(TagType t) { @@ -81,9 +81,8 @@ namespace reindexer { return ""sv; case TAG_UUID: return ""sv; - default: - throw Error{errNotValid, "Invalid tag type: %d", t}; } + throw Error{errNotValid, "Invalid tag type: %d", t}; } [[nodiscard]] std::string_view AggTypeToStr(AggType t) noexcept { @@ -106,9 +105,26 @@ namespace reindexer { case AggCountCached: return "count_cached"sv; case AggUnknown: - default: - return "unknown"sv; + break; + } + return "unknown"sv; +} + +[[nodiscard]] std::string_view JoinTypeName(JoinType type) { + using namespace std::string_view_literals; + + switch (type) { + case JoinType::InnerJoin: + return "INNER JOIN"sv; + case JoinType::OrInnerJoin: + return "OR INNER JOIN"sv; + case JoinType::LeftJoin: + return "LEFT JOIN"sv; + case JoinType::Merge: + return "MERGE"sv; } + assertrx(false); + return "unknown"sv; } } // namespace reindexer diff --git a/cpp_src/core/type_consts_helpers.h b/cpp_src/core/type_consts_helpers.h index c5aa63ce2..2fc5e36df 100644 --- a/cpp_src/core/type_consts_helpers.h +++ b/cpp_src/core/type_consts_helpers.h @@ -11,6 +11,11 @@ namespace reindexer { [[nodiscard]] std::string_view TagTypeToStr(TagType); [[nodiscard]] std::string_view AggTypeToStr(AggType t) noexcept; +/// Get readable Join Type +/// @param type - join type +/// @returns string with join type name +[[nodiscard]] std::string_view JoinTypeName(JoinType type); + template auto& operator<<(T& os, CondType cond) { switch (cond) { @@ -38,9 +43,8 @@ auto& operator<<(T& os, CondType cond) { return os << "LIKE"; case CondDWithin: return os << "DWITHIN"; - default: - abort(); } + std::abort(); } template @@ -52,25 +56,13 @@ auto& operator<<(T& os, OpType op) { return os << "AND"; case OpNot: return os << "NOT"; - default: - abort(); } + std::abort(); } template auto& operator<<(T& os, JoinType jt) { - switch (jt) { - case LeftJoin: - return os << "LEFT JOIN"; - case InnerJoin: - return os << "INNER JOIN"; - case OrInnerJoin: - return os << "OR INNER JOIN"; - case Merge: - return os << "MERGE"; - default: - abort(); - } + return os << JoinTypeName(jt); } template @@ -118,9 +110,10 @@ T& operator<<(T& os, IndexType it) { return os << "RTree"; case IndexUuidHash: return os << "UuidHash"; - default: - abort(); + case IndexUuidStore: + return os << "UuidStore"; } + std::abort(); } template @@ -136,9 +129,8 @@ T& operator<<(T& os, CollateMode m) { return os << "Numeric"; case CollateCustom: return os << "Custom"; - default: - abort(); } + std::abort(); } constexpr bool IsComposite(IndexType type) noexcept { diff --git a/cpp_src/debug/allocdebug.h b/cpp_src/debug/allocdebug.h index 3425aa40c..a0d44f32b 100644 --- a/cpp_src/debug/allocdebug.h +++ b/cpp_src/debug/allocdebug.h @@ -1,5 +1,5 @@ #pragma once -#include +#include void allocdebug_show(); void allocdebug_init(); diff --git a/cpp_src/debug/backtrace.cc b/cpp_src/debug/backtrace.cc index fdb4a99ef..2c74393ba 100644 --- a/cpp_src/debug/backtrace.cc +++ b/cpp_src/debug/backtrace.cc @@ -183,7 +183,7 @@ static void sighandler(int sig, siginfo_t *, void *ctx) { exit(-1); } -void backtrace_init() { +void backtrace_init() noexcept { struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_sigaction = sighandler; @@ -220,7 +220,7 @@ static std::recursive_mutex g_mutex; static crash_query_reporter_t g_crash_query_reporter = [](std::ostream &) {}; static backtrace_writer_t g_writer = [](std::string_view sv) { std::cerr << sv; }; -void backtrace_init() {} +void backtrace_init() noexcept {} void backtrace_set_writer(backtrace_writer_t) {} int backtrace_internal(void **, size_t, void *, std::string_view &) { return 0; } void backtrace_set_crash_query_reporter(crash_query_reporter_t reporter) { diff --git a/cpp_src/debug/backtrace.h b/cpp_src/debug/backtrace.h index 3750b2ba0..416616f49 100644 --- a/cpp_src/debug/backtrace.h +++ b/cpp_src/debug/backtrace.h @@ -9,7 +9,7 @@ namespace debug { using backtrace_writer_t = std::function; using crash_query_reporter_t = std::function; -void backtrace_init(); +void backtrace_init() noexcept; void backtrace_set_writer(backtrace_writer_t); void backtrace_set_crash_query_reporter(crash_query_reporter_t); backtrace_writer_t backtrace_get_writer(); diff --git a/cpp_src/debug/terminate_handler.cpp b/cpp_src/debug/terminate_handler.cpp index a44a0de2d..7269ccde5 100644 --- a/cpp_src/debug/terminate_handler.cpp +++ b/cpp_src/debug/terminate_handler.cpp @@ -11,7 +11,7 @@ namespace reindexer { namespace debug { -void terminate_handler() { +static void terminate_handler() { std::ostringstream sout; std::exception_ptr exptr = std::current_exception(); if (exptr) { diff --git a/cpp_src/estl/defines.h b/cpp_src/estl/defines.h index e3d1b67ed..118755a90 100644 --- a/cpp_src/estl/defines.h +++ b/cpp_src/estl/defines.h @@ -5,7 +5,7 @@ #if defined(__GNUC__) || defined(__clang__) #define rx_likely(x) (__builtin_expect(!!(x), 1)) #else // defined(__GNUC__) || defined(__clang__) -#define rx_likely(x) x +#define rx_likely(x) (x) #endif // defined(__GNUC__) || defined(__clang__) #else // !defined(likely) #define rx_likely(x) likely(x) @@ -16,7 +16,7 @@ #if defined(__GNUC__) || defined(__clang__) #define rx_unlikely(x) (__builtin_expect(!!(x), 0)) #else // defined(__GNUC__) || defined(__clang__) -#define rx_unlikely(x) x +#define rx_unlikely(x) (x) #endif // defined(__GNUC__) || defined(__clang__) #else // !defined(unlikely) #define rx_unlikely(x) unlikely(x) diff --git a/cpp_src/estl/intrusive_ptr.h b/cpp_src/estl/intrusive_ptr.h index 3e8f6c320..41faaaa52 100644 --- a/cpp_src/estl/intrusive_ptr.h +++ b/cpp_src/estl/intrusive_ptr.h @@ -4,6 +4,8 @@ #include #include "tools/assertrx.h" +#include + namespace reindexer { template @@ -156,21 +158,21 @@ template class intrusive_atomic_rc_wrapper; template -inline static void intrusive_ptr_add_ref(intrusive_atomic_rc_wrapper *x) noexcept { +inline void intrusive_ptr_add_ref(intrusive_atomic_rc_wrapper *x) noexcept { if (x) { x->refcount.fetch_add(1, std::memory_order_relaxed); } } template -inline static void intrusive_ptr_release(intrusive_atomic_rc_wrapper *x) noexcept { +inline void intrusive_ptr_release(intrusive_atomic_rc_wrapper *x) noexcept { if (x && x->refcount.fetch_sub(1, std::memory_order_acq_rel) == 1) { delete x; } } template -inline static bool intrusive_ptr_is_unique(intrusive_atomic_rc_wrapper *x) noexcept { +inline bool intrusive_ptr_is_unique(intrusive_atomic_rc_wrapper *x) noexcept { // std::memory_order_acquire - is essential for COW constructions based on intrusive_ptr return !x || (x->refcount.load(std::memory_order_acquire) == 1); } @@ -190,6 +192,37 @@ class intrusive_atomic_rc_wrapper : public T { friend bool intrusive_ptr_is_unique<>(intrusive_atomic_rc_wrapper *x) noexcept; }; +class intrusive_atomic_rc_base { +public: + intrusive_atomic_rc_base() noexcept : refcount(0) {} + intrusive_atomic_rc_base &operator=(const intrusive_atomic_rc_base &) = delete; + virtual ~intrusive_atomic_rc_base() = default; + +protected: + std::atomic refcount; + + friend void intrusive_ptr_add_ref(intrusive_atomic_rc_base *x) noexcept; + friend void intrusive_ptr_release(intrusive_atomic_rc_base *x) noexcept; + friend bool intrusive_ptr_is_unique(intrusive_atomic_rc_base *x) noexcept; +}; + +inline void intrusive_ptr_add_ref(intrusive_atomic_rc_base *x) noexcept { + if (x) { + x->refcount.fetch_add(1, std::memory_order_relaxed); + } +} + +inline void intrusive_ptr_release(intrusive_atomic_rc_base *x) noexcept { + if (x && x->refcount.fetch_sub(1, std::memory_order_acq_rel) == 1) { + delete x; + } +} + +inline bool intrusive_ptr_is_unique(intrusive_atomic_rc_base *x) noexcept { + // std::memory_order_acquire - is essential for COW constructions based on intrusive_ptr + return !x || (x->refcount.load(std::memory_order_acquire) == 1); +} + template intrusive_ptr make_intrusive(Args &&...args) { return intrusive_ptr(new T(std::forward(args)...)); diff --git a/cpp_src/estl/tokenizer.cc b/cpp_src/estl/tokenizer.cc index d4986ccc8..ba3e27f75 100644 --- a/cpp_src/estl/tokenizer.cc +++ b/cpp_src/estl/tokenizer.cc @@ -1,14 +1,9 @@ #include "tokenizer.h" -#include #include "tools/stringstools.h" namespace reindexer { -tokenizer::tokenizer(std::string_view query) : q_(query), cur_(query.begin()) {} - -bool tokenizer::end() const { return cur_ == q_.end(); } - -void tokenizer::skip_space() { +void tokenizer::skip_space() noexcept { for (;;) { while (cur_ != q_.end() && (*cur_ == ' ' || *cur_ == '\t' || *cur_ == '\n')) { cur_++; @@ -26,7 +21,7 @@ void tokenizer::skip_space() { } } -token tokenizer::next_token(bool to_lower, bool treatSignAsToken, bool inOrderBy) { +token tokenizer::next_token(flags flgs) { skip_space(); if (cur_ == q_.end()) return token(TokenEnd); @@ -38,14 +33,14 @@ token tokenizer::next_token(bool to_lower, bool treatSignAsToken, bool inOrderBy int openBrackets{0}; do { if (*cur_ == '*' && *(cur_ - 1) != '[') break; - res.text_.push_back(to_lower ? tolower(*cur_++) : *cur_++); + res.text_.push_back(flgs.has_to_lower() ? tolower(*cur_++) : *cur_++); ++pos_; } while (cur_ != q_.end() && (isalpha(*cur_) || isdigit(*cur_) || *cur_ == '_' || *cur_ == '#' || *cur_ == '.' || *cur_ == '*' || (*cur_ == '[' && (++openBrackets, true)) || (*cur_ == ']' && (--openBrackets >= 0)))); } else if (*cur_ == '"') { res.type = TokenName; const size_t startPos = ++pos_; - if (inOrderBy) { + if (flgs.has_in_order_by()) { res.text_.push_back('"'); } while (++cur_ != q_.end() && *cur_ != '"') { @@ -58,10 +53,10 @@ token tokenizer::next_token(bool to_lower, bool treatSignAsToken, bool inOrderBy !isalpha(*cur_) && !isdigit(*cur_)) { throw Error{errParseSQL, "Identifier should not contain '%c'; %s", *cur_, where()}; } - res.text_.push_back(to_lower ? tolower(*cur_) : *cur_); + res.text_.push_back(flgs.has_to_lower() ? tolower(*cur_) : *cur_); ++pos_; } - if (inOrderBy) { + if (flgs.has_in_order_by()) { res.text_.push_back('"'); } if (cur_ == q_.end()) { @@ -69,13 +64,13 @@ token tokenizer::next_token(bool to_lower, bool treatSignAsToken, bool inOrderBy } ++cur_; ++pos_; - } else if (isdigit(*cur_) || (!treatSignAsToken && (*cur_ == '-' || *cur_ == '+'))) { + } else if (isdigit(*cur_) || (!flgs.has_treat_sign_as_token() && (*cur_ == '-' || *cur_ == '+'))) { res.type = TokenNumber; do { res.text_.push_back(*cur_++); ++pos_; } while (cur_ != q_.end() && (isdigit(*cur_) || *cur_ == '.')); - } else if (treatSignAsToken && (*cur_ == '-' || *cur_ == '+')) { + } else if (flgs.has_treat_sign_as_token() && (*cur_ == '-' || *cur_ == '+')) { res.type = TokenSign; res.text_.push_back(*cur_++); ++pos_; @@ -121,8 +116,9 @@ token tokenizer::next_token(bool to_lower, bool treatSignAsToken, bool inOrderBy } } res.text_.push_back(c); - ++pos_, ++cur_; - }; + ++pos_; + ++cur_; + } } else { res.text_.push_back(*cur_++); ++pos_; @@ -145,26 +141,13 @@ std::string tokenizer::where() const { } else col++; } - return "line: " + std::to_string(line) + " column: " + std::to_string(col) + " " + std::to_string(q_.size()); -} - -token tokenizer::peek_token(bool to_lower, bool treatSignAsToken, bool inOrderBy) { - auto save_cur = cur_; - auto save_pos = pos_; - auto res = next_token(to_lower, treatSignAsToken, inOrderBy); - cur_ = save_cur; - pos_ = save_pos; - return res; + return std::string() + .append("line: ") + .append(std::to_string(line)) + .append(" column: ") + .append(std::to_string(col)) + .append(" ") + .append(std::to_string(q_.size())); } -void tokenizer::setPos(size_t pos) { - int delta = pos - pos_; - pos_ += delta; - cur_ += delta; -} -size_t tokenizer::getPos() const { return pos_; } - -size_t tokenizer::length() const { return q_.length(); } -const char *tokenizer::begin() const { return q_.data(); } - } // namespace reindexer diff --git a/cpp_src/estl/tokenizer.h b/cpp_src/estl/tokenizer.h index 01dce455c..399083e19 100644 --- a/cpp_src/estl/tokenizer.h +++ b/cpp_src/estl/tokenizer.h @@ -1,5 +1,3 @@ - - #pragma once #include @@ -12,42 +10,60 @@ enum token_type { TokenEnd, TokenName, TokenNumber, TokenString, TokenOp, TokenS class token { public: - token() : type(TokenSymbol) {} - token(token_type type) : type(type) {} + token(token_type t = TokenSymbol) noexcept : type(t) {} token(const token &) = delete; token &operator=(const token &) = delete; - token(token &&other) : type(other.type), text_(std::move(other.text_)) { - text_.reserve(other.text_.size() + 1); - *(text_.begin() + text_.size()) = 0; - } - token &operator=(token &&other) { - if (&other != this) { - type = other.type; - text_ = std::move(other.text_); - text_.reserve(other.text_.size() + 1); - *(text_.begin() + text_.size()) = 0; + token(token &&other) = default; + token &operator=(token &&other) = default; + + std::string_view text() const noexcept { return std::string_view(text_.data(), text_.size()); } + void to_lower() noexcept { + for (auto &c : text_) { + c = tolower(c); } - return *this; } - std::string_view text() const { return std::string_view(text_.data(), text_.size()); } - token_type type; h_vector text_; }; class tokenizer { public: - tokenizer(std::string_view query); - token next_token(bool to_lower = true, bool treatSignAsToken = false, bool inOrderBy = false); - token peek_token(bool to_lower = true, bool treatSignAsToken = false, bool inOrderBy = false); - void skip_space(); - bool end() const; - size_t getPos() const; - void setPos(size_t pos); + class flags { + public: + enum values : int { no_flags = 0, to_lower = 1, treat_sign_as_token = 1 << 1, in_order_by = 1 << 2 }; + + flags(int f) noexcept : f_(f) {} + + RX_ALWAYS_INLINE bool has_to_lower() const noexcept { return f_ & values::to_lower; } + RX_ALWAYS_INLINE bool has_treat_sign_as_token() const noexcept { return f_ & values::treat_sign_as_token; } + RX_ALWAYS_INLINE bool has_in_order_by() const noexcept { return f_ & values::in_order_by; } + + private: + int f_; + }; + + tokenizer(std::string_view query) noexcept : q_(query), cur_(query.begin()) {} + token next_token(flags f = flags(flags::to_lower)); + token peek_token(flags f = flags(flags::to_lower)) { + auto save_cur = cur_; + auto save_pos = pos_; + auto res = next_token(f); + cur_ = save_cur; + pos_ = save_pos; + return res; + } + void skip_space() noexcept; + bool end() const noexcept { return cur_ == q_.end(); } + size_t getPos() const noexcept { return pos_; } + void setPos(size_t pos) noexcept { + int delta = pos - pos_; + pos_ += delta; + cur_ += delta; + } std::string where() const; - size_t length() const; - const char *begin() const; + size_t length() const noexcept { return q_.length(); } + const char *begin() const noexcept { return q_.data(); } protected: std::string_view q_; diff --git a/cpp_src/gtests/tests/API/base_tests.cc b/cpp_src/gtests/tests/API/base_tests.cc index 2cceb9bd1..a466ceb28 100644 --- a/cpp_src/gtests/tests/API/base_tests.cc +++ b/cpp_src/gtests/tests/API/base_tests.cc @@ -1616,11 +1616,13 @@ TEST_F(ReindexerApi, LoggerWriteInterruptTest) { std::shared_ptr sinkPtr; } instance; - reindexer::logInstallWriter([&](int level, char* buf) { - if (level <= LogTrace) { - instance.logger.trace(buf); - } - }); + reindexer::logInstallWriter( + [&](int level, char* buf) { + if (level <= LogTrace) { + instance.logger.trace(buf); + } + }, + reindexer::LoggerPolicy::WithLocks); auto writeThread = std::thread([]() { for (size_t i = 0; i < 10000; ++i) { reindexer::logPrintf(LogTrace, "Detailed and amazing description of this error: [%d]!", i); @@ -1636,7 +1638,7 @@ TEST_F(ReindexerApi, LoggerWriteInterruptTest) { writeThread.join(); reopenThread.join(); reindexer::logPrintf(LogTrace, "FINISHED\n"); - reindexer::logInstallWriter(nullptr); + reindexer::logInstallWriter(nullptr, reindexer::LoggerPolicy::WithLocks); } TEST_F(ReindexerApi, IntToStringIndexUpdate) { @@ -1719,3 +1721,83 @@ TEST_F(ReindexerApi, SelectFilterWithAggregationConstraints) { EXPECT_NO_THROW(Query().FromSQL("select *, count(*) from test_namespace")); EXPECT_NO_THROW(Query().FromSQL("select count(*), * from test_namespace")); } + +TEST_F(ReindexerApi, InsertIncorrectItemWithJsonPathsDuplication) { + Error err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); + ASSERT_TRUE(err.ok()) << err.what(); + + err = rt.reindexer->AddIndex(default_namespace, {"id", "hash", "int", IndexOpts().PK()}); + ASSERT_TRUE(err.ok()) << err.what(); + + Item oldTagsItemCJSON = rt.reindexer->NewItem(default_namespace); + ASSERT_TRUE(oldTagsItemCJSON.Status().ok()) << oldTagsItemCJSON.Status().what(); + Item oldTagsItemJSON = rt.reindexer->NewItem(default_namespace); + ASSERT_TRUE(oldTagsItemJSON.Status().ok()) << oldTagsItemJSON.Status().what(); + + err = rt.reindexer->AddIndex(default_namespace, {"value", reindexer::JsonPaths{"value1"}, "hash", "string", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + + { + // Check item unmarshaled from json + Item item(rt.reindexer->NewItem(default_namespace)); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + err = item.FromJSON(R"_({"id":0,"value1":"v","obj":{"id":11},"value1":"p"})_"); + EXPECT_EQ(err.code(), errLogic) << err.what(); + } + { + // Check item unmarshaled from cjson (with correct tags) + Item item(rt.reindexer->NewItem(default_namespace)); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + constexpr char cjson[] = {0x06, 0x08, 0x00, 0x12, 0x01, 0x70, 0x12, 0x01, 0x70, 0x07}; + err = item.FromCJSON(std::string_view(cjson, sizeof(cjson))); + ASSERT_EQ(err.code(), errLogic); + } + { + // Check item unmarshaled from msgpack + Item item(rt.reindexer->NewItem(default_namespace)); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + constexpr uint8_t msgpack[] = {0xDF, 0x00, 0x00, 0x00, 0x04, 0xA2, 0x69, 0x64, 0x00, 0xA6, 0x76, 0x61, 0x6C, 0x75, + 0x65, 0x31, 0xA1, 0x76, 0xA3, 0x6F, 0x62, 0x6A, 0xDF, 0x00, 0x00, 0x00, 0x01, 0xA2, + 0x69, 0x64, 0x0B, 0xA6, 0x76, 0x61, 0x6C, 0x75, 0x65, 0x31, 0xA1, 0x70}; + size_t offset = 0; + err = item.FromMsgPack(std::string_view(reinterpret_cast(msgpack), sizeof(msgpack)), offset); + EXPECT_EQ(err.code(), errLogic) << err.what(); + } + { + // Check item unmarshaled from msgpack (different encoding) + Item item(rt.reindexer->NewItem(default_namespace)); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + constexpr uint8_t msgpack[] = {0x84, 0xA2, 0x69, 0x64, 0x00, 0xA6, 0x76, 0x61, 0x6C, 0x75, 0x65, 0x31, 0xA1, 0x70, 0xA3, 0x6F, + 0x62, 0x6A, 0x81, 0xA2, 0x69, 0x64, 0x0B, 0xA6, 0x76, 0x61, 0x6C, 0x75, 0x65, 0x31, 0xA1, 0x70}; + size_t offset = 0; + err = item.FromMsgPack(std::string_view(reinterpret_cast(msgpack), sizeof(msgpack)), offset); + EXPECT_EQ(err.code(), errLogic) << err.what(); + } + + { + // Check item unmarshaled from cjson (with outdated tags) + constexpr char cjson[] = {0x07, 0x13, 0x00, 0x00, 0x00, 0x06, 0x08, 0x00, 0x12, 0x01, 0x76, 0x1E, 0x08, 0x16, 0x07, 0x12, 0x01, + 0x70, 0x07, 0x03, 0x02, 0x69, 0x64, 0x06, 0x76, 0x61, 0x6C, 0x75, 0x65, 0x31, 0x03, 0x6F, 0x62, 0x6A}; + err = oldTagsItemCJSON.FromCJSON(std::string_view(cjson, sizeof(cjson))); + EXPECT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->Insert(default_namespace, oldTagsItemCJSON); + ASSERT_EQ(err.code(), errLogic); + + QueryResults qr; + err = rt.reindexer->Select(Query(default_namespace), qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.Count(), 0); + } + { + // Check item unmarshaled from json with outdated tags + err = oldTagsItemJSON.FromJSON(R"_({"id":0,"value1":"v","obj":{"id":11},"value1":"p"})_"); + EXPECT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->Insert(default_namespace, oldTagsItemJSON); + ASSERT_EQ(err.code(), errLogic); + + QueryResults qr; + err = rt.reindexer->Select(Query(default_namespace), qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.Count(), 0); + } +} diff --git a/cpp_src/gtests/tests/fixtures/join_selects_api.h b/cpp_src/gtests/tests/fixtures/join_selects_api.h index 0003c6345..cdf0f9189 100644 --- a/cpp_src/gtests/tests/fixtures/join_selects_api.h +++ b/cpp_src/gtests/tests/fixtures/join_selects_api.h @@ -390,6 +390,21 @@ class JoinSelectsApi : public ReindexerApi { EXPECT_TRUE(pagesConditionResult || joinsBracketConditionsResult || priceConditionResult || joinsNoBracketConditionsResult); } } + void ValidateQueryError(std::string_view sql, ErrorCode expectedCode, std::string_view expectedText) { + QueryResults qr; + { + auto err = rt.reindexer->Select(sql, qr); + EXPECT_EQ(err.code(), expectedCode) << sql; + EXPECT_EQ(err.what(), expectedText) << sql; + } + { + Query q; + q.FromSQL(sql); + auto err = rt.reindexer->Select(q, qr); + EXPECT_EQ(err.code(), expectedCode) << sql; + EXPECT_EQ(err.what(), expectedText) << sql; + } + } static std::string addQuotes(const std::string& str) { std::string output; diff --git a/cpp_src/gtests/tests/fixtures/ns_api.h b/cpp_src/gtests/tests/fixtures/ns_api.h index b162547c3..c2e8146ba 100644 --- a/cpp_src/gtests/tests/fixtures/ns_api.h +++ b/cpp_src/gtests/tests/fixtures/ns_api.h @@ -96,6 +96,28 @@ class NsApi : public ReindexerApi { } } + void CreateEmptyArraysNamespace(const std::string& nsName) { + Error err = rt.reindexer->OpenNamespace(nsName); + ASSERT_TRUE(err.ok()) << err.what(); + + DefineNamespaceDataset(nsName, {IndexDeclaration{idIdxName.c_str(), "hash", "int", IndexOpts().PK(), 0}, + IndexDeclaration{indexedArrayField.c_str(), "hash", "int", IndexOpts().Array(), 0}}); + + char sourceJson[1024]; + const char jsonPattern[] = R"json({"id": %s, "indexed_array_field": [], "non_indexed_array_field": []})json"; + for (size_t i = 100; i < 200; ++i) { + Item item = NewItem(nsName); + EXPECT_TRUE(item.Status().ok()) << item.Status().what(); + + std::string serial = std::to_string(i); + snprintf(sourceJson, sizeof(sourceJson) - 1, jsonPattern, serial.c_str()); + + err = item.FromJSON(sourceJson); + EXPECT_TRUE(err.ok()) << err.what(); + Upsert(nsName, item); + } + } + void InsertNewTruncateItem(int i) { Item item = NewItem(truncate_namespace); item[idIdxName] = i; @@ -163,6 +185,15 @@ class NsApi : public ReindexerApi { } } + void AddItemFromJSON(std::string_view ns, std::string_view json) { + Item item = NewItem(ns); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + auto err = item.FromJSON(json); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->Insert(ns, item); + ASSERT_TRUE(err.ok()) << err.what(); + } + const std::string truncate_namespace = "truncate_namespace"; const std::string idIdxName = "id"; const std::string updatedTimeSecFieldName = "updated_time_sec"; diff --git a/cpp_src/gtests/tests/fixtures/queries_api.cc b/cpp_src/gtests/tests/fixtures/queries_api.cc index 98b5a7802..9047a9627 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.cc +++ b/cpp_src/gtests/tests/fixtures/queries_api.cc @@ -71,6 +71,196 @@ void QueriesApi::CheckMergeQueriesWithLimit() { EXPECT_EQ(qr.getMergedNSCount(), 2); } +void QueriesApi::CheckMergeQueriesWithAggregation() { + auto AggSelect = [this](const Query& q, AggType tp, double& val) -> void { + QueryResults qr; + Error err = rt.reindexer->Select(q, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.aggregationResults.size(), 1); + ASSERT_TRUE(qr.aggregationResults[0].GetValue().has_value()); + ASSERT_EQ(qr.aggregationResults[0].type, tp); + val = qr.aggregationResults[0].GetValue().value(); + if (tp == AggCount || tp == AggCountCached) { + ASSERT_EQ(val, qr.totalCount); + } + }; + // check the correctness of the aggregation functions with the merge query + { + double c1; + AggSelect(Query{default_namespace}.Aggregate(AggCount, {}), AggCount, c1); + double c2; + AggSelect(Query{joinNs}.Aggregate(AggCount, {}), AggCount, c2); + double c3; + AggSelect(Query{default_namespace}.Aggregate(AggCount, {}).Merge(Query{joinNs}), AggCount, c3); + double c4; + AggSelect(Query{testSimpleNs}.Aggregate(AggCount, {}), AggCount, c4); + double c5; + AggSelect(Query{default_namespace}.Aggregate(AggCount, {}).Merge(Query{joinNs}).Merge(Query{testSimpleNs}), AggCount, c5); + { + QueryResults qr; + Error err = rt.reindexer->Select(Query{default_namespace}.ReqTotal().Merge(Query{joinNs}).Merge(Query{testSimpleNs}), qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.totalCount, c5); + } + ASSERT_EQ(c1 + c2, c3); + ASSERT_EQ(c1 + c2 + c4, c5); + } + + { + double c1; + AggSelect(Query{default_namespace}.Aggregate(AggCount, {}), AggCount, c1); + double c2; + AggSelect(Query{joinNs}.Aggregate(AggCount, {}), AggCount, c2); + double c4; + AggSelect(Query{testSimpleNs}.Aggregate(AggCount, {}), AggCount, c4); + double c3; + Query q3; + q3.FromSQL(fmt::sprintf("SELECT count(*) FROM %s MERGE (SELECT * FROM %s)", default_namespace, joinNs)); + AggSelect(q3, AggCount, c3); + double c5; + Query q5; + q5.FromSQL(fmt::sprintf("SELECT count(*) FROM %s MERGE (SELECT * FROM %s) MERGE (SELECT * FROM %s)", default_namespace, joinNs, + testSimpleNs)); + AggSelect(q5, AggCount, c5); + { + QueryResults qr; + Error err = rt.reindexer->Select(Query{default_namespace}.CachedTotal().Merge(Query{joinNs}).Merge(Query{testSimpleNs}), qr); + ASSERT_EQ(qr.totalCount, c5); + } + ASSERT_EQ(c1 + c2, c3); + ASSERT_EQ(c1 + c2 + c4, c5); + } + + { + double c1; + AggSelect(Query{default_namespace}.Aggregate(AggCountCached, {}), AggCountCached, c1); + double c2; + AggSelect(Query{joinNs}.Aggregate(AggCountCached, {}), AggCountCached, c2); + double c3; + AggSelect(Query{default_namespace}.Aggregate(AggCountCached, {}).Merge(Query{joinNs}), AggCountCached, c3); + double c4; + AggSelect(Query{testSimpleNs}.Aggregate(AggCountCached, {}), AggCountCached, c4); + double c5; + AggSelect(Query{default_namespace}.Aggregate(AggCountCached, {}).Merge(Query{joinNs}).Merge(Query{testSimpleNs}), AggCountCached, c5); + ASSERT_EQ(c1 + c2, c3); + ASSERT_EQ(c1 + c2 + c4, c5); + } + + { + double c1; + AggSelect(Query{default_namespace}.Aggregate(AggMin, {"id"}), AggMin, c1); + double c2; + AggSelect(Query{joinNs}.Aggregate(AggMin, {"id"}), AggMin, c2); + double c3; + AggSelect(Query{default_namespace}.Aggregate(AggMin, {"id"}).Merge(Query{joinNs}), AggMin, c3); + double c4; + AggSelect(Query{testSimpleNs}.Aggregate(AggMin, {"id"}), AggMin, c4); + double c5; + AggSelect(Query{default_namespace}.Aggregate(AggMin, {"id"}).Merge(Query{joinNs}).Merge(Query{testSimpleNs}), AggMin, c5); + ASSERT_EQ(std::min(c1, c2), c3); + ASSERT_EQ(std::min({c1, c2, c4}), c5); + } + { + double c1; + AggSelect(Query{default_namespace}.Aggregate(AggMax, {"id"}), AggMax, c1); + double c2; + AggSelect(Query{joinNs}.Aggregate(AggMax, {"id"}), AggMax, c2); + double c3; + AggSelect(Query{default_namespace}.Aggregate(AggMax, {"id"}).Merge(Query{joinNs}), AggMax, c3); + double c4; + AggSelect(Query{testSimpleNs}.Aggregate(AggMax, {"id"}), AggMax, c4); + double c5; + AggSelect(Query{default_namespace}.Aggregate(AggMax, {"id"}).Merge(Query{joinNs}).Merge(Query{testSimpleNs}), AggMax, c5); + ASSERT_EQ(std::max(c1, c2), c3); + ASSERT_EQ(std::max({c1, c2, c4}), c5); + } + + { + double c1; + AggSelect(Query{default_namespace}.Aggregate(AggSum, {"id"}), AggSum, c1); + double c2; + AggSelect(Query{joinNs}.Aggregate(AggSum, {"id"}), AggSum, c2); + double c3; + AggSelect(Query{default_namespace}.Aggregate(AggSum, {"id"}).Merge(Query{joinNs}), AggSum, c3); + double c4; + AggSelect(Query{testSimpleNs}.Aggregate(AggSum, {"id"}), AggSum, c4); + double c5; + AggSelect(Query{default_namespace}.Aggregate(AggSum, {"id"}).Merge(Query{joinNs}).Merge(Query{testSimpleNs}), AggSum, c5); + ASSERT_EQ(c1 + c2, c3); + ASSERT_EQ(c1 + c2 + c4, c5); + } + // check the correctness of the error for unsupported cases + { + QueryResults qr; + Error err = rt.reindexer->Select(Query{default_namespace}.Aggregate(AggSum, {"id"}).Limit(10).Offset(10).Merge(Query{joinNs}), qr); + EXPECT_FALSE(err.ok()); + EXPECT_EQ(err.what(), "Limit and offset are not supported for aggregations 'sum'"); + } + { + QueryResults qr; + Error err = rt.reindexer->Select(Query{default_namespace}.Aggregate(AggMin, {"id"}).Limit(10).Merge(Query{joinNs}), qr); + EXPECT_FALSE(err.ok()); + EXPECT_EQ(err.what(), "Limit and offset are not supported for aggregations 'min'"); + } + { + QueryResults qr; + Error err = rt.reindexer->Select(Query{default_namespace}.Aggregate(AggMax, {"id"}).Offset(10).Merge(Query{joinNs}), qr); + EXPECT_FALSE(err.ok()); + EXPECT_EQ(err.what(), "Limit and offset are not supported for aggregations 'max'"); + } + { + QueryResults qr; + Error err = rt.reindexer->Select(Query{default_namespace}.Merge(Query{joinNs}.ReqTotal()), qr); + EXPECT_FALSE(err.ok()); + EXPECT_EQ(err.what(), "Aggregations in inner merge query is not allowed"); + } + { + QueryResults qr; + Error err = rt.reindexer->Select(Query{default_namespace}.Merge(Query{joinNs}.CachedTotal()), qr); + EXPECT_FALSE(err.ok()); + EXPECT_EQ(err.what(), "Aggregations in inner merge query is not allowed"); + } + // checking the work of several aggregate functions with the merge query + { + Query q = Query{default_namespace} + .Aggregate(AggSum, {"id"}) + .Aggregate(AggCount, {}) + .Aggregate(AggMin, {"id"}) + .Merge(Query{joinNs}) + .Merge(Query{testSimpleNs}); + QueryResults qr; + Error err = rt.reindexer->Select(q, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.aggregationResults.size(), 3); + for (auto a : {AggSum, AggCount, AggMin}) { + int exist = 0; + for (const auto& ar : qr.aggregationResults) { + if (ar.type == a) { + exist++; + } + } + ASSERT_EQ(exist, 1); + } + } + { + double c1; + AggSelect(Query{default_namespace}.Aggregate(AggCount, {}), AggCount, c1); + double c2; + AggSelect(Query{joinNs}.Aggregate(AggCount, {}), AggCount, c2); + double c3; + AggSelect(Query{testSimpleNs}.Aggregate(AggCount, {}), AggCount, c3); + + Query q = + Query{default_namespace}.Aggregate(AggCount, {}).Aggregate(AggCountCached, {}).Merge(Query{joinNs}).Merge(Query{testSimpleNs}); + QueryResults qr; + Error err = rt.reindexer->Select(q, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.aggregationResults.size(), 1); + ASSERT_EQ(qr.aggregationResults[0].type, AggCount); + ASSERT_EQ(qr.aggregationResults[0].GetValueOrZero(), c1 + c2 + c3); + } +} + static struct { reindexer::KeyValueType fieldType; std::vector indexTypes; diff --git a/cpp_src/gtests/tests/fixtures/queries_api.h b/cpp_src/gtests/tests/fixtures/queries_api.h index eea582552..20e4aa9af 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.h +++ b/cpp_src/gtests/tests/fixtures/queries_api.h @@ -596,6 +596,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { } void CheckMergeQueriesWithLimit(); + void CheckMergeQueriesWithAggregation(); void CheckGeomQueries() { for (size_t i = 0; i < 10; ++i) { @@ -1461,6 +1462,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Distinct(distinct)); for (CondType cond : {CondEq, CondSet, CondLt, CondLe, CondGt, CondGe}) { + const bool multyArgCond = cond == CondEq || cond == CondSet; ExecuteAndVerify(Query(default_namespace) .Where(kFieldNameUuid, cond, randUuid()) .Distinct(distinct.c_str()) @@ -1471,12 +1473,13 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); - ExecuteAndVerify( - Query(default_namespace) - .Where(kFieldNameUuid, cond, - VariantArray::Create(randUuid(), randStrUuid(), randUuid(), randStrUuid(), randUuid())) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuid, cond, + multyArgCond ? VariantArray::Create(randUuid(), randStrUuid(), randUuid(), + randStrUuid(), randUuid()) + : VariantArray::Create(randUuid())) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); ExecuteAndVerify(Query(default_namespace) .Where(kFieldNameUuidArr, cond, randUuid()) @@ -1490,27 +1493,37 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { ExecuteAndVerify(Query(default_namespace) .Where(kFieldNameUuidArr, cond, - VariantArray::Create(randUuid(), randStrUuid(), randUuid(), randStrUuid(), - randUuid(), randStrUuid())) + multyArgCond ? VariantArray::Create(randUuid(), randStrUuid(), randUuid(), + randStrUuid(), randUuid(), randStrUuid()) + : VariantArray::Create(randUuid())) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); - ExecuteAndVerify( - Query(default_namespace) - .WhereComposite( - kFieldNameUuid + compositePlus + kFieldNameName, cond, - {VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString())}) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); + ExecuteAndVerify(Query(default_namespace) + .WhereComposite(kFieldNameUuid + compositePlus + kFieldNameName, cond, + multyArgCond + ? std::vector{VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString()), + VariantArray::Create(randStrUuid(), RandString()), + VariantArray::Create(randUuid(), RandString())} + : std::vector{VariantArray::Create(randUuid(), RandString())}) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); } ExecuteAndVerify(Query(default_namespace) diff --git a/cpp_src/gtests/tests/fixtures/ttl_index_api.h b/cpp_src/gtests/tests/fixtures/ttl_index_api.h index 662e202ce..9e6768fef 100644 --- a/cpp_src/gtests/tests/fixtures/ttl_index_api.h +++ b/cpp_src/gtests/tests/fixtures/ttl_index_api.h @@ -60,7 +60,7 @@ class TtlIndexApi : public ReindexerApi { #if !defined(REINDEXER_WITH_TSAN) && !defined(REINDEX_WITH_ASAN) constexpr auto kStep = std::chrono::milliseconds(100); #else // !defined (REINDEXER_WITH_TSAN) && !defined(REINDEX_WITH_ASAN) - constexpr auto kStep = std::chrono::milliseconds(200); + constexpr auto kStep = std::chrono::milliseconds(300); #endif // !defined (REINDEXER_WITH_TSAN) && !defined(REINDEX_WITH_ASAN) size_t count = GetItemsCount(); if (count > 0) { diff --git a/cpp_src/gtests/tests/unit/composite_indexes_api.h b/cpp_src/gtests/tests/unit/composite_indexes_api.h index e1813704d..a555bb162 100644 --- a/cpp_src/gtests/tests/unit/composite_indexes_api.h +++ b/cpp_src/gtests/tests/unit/composite_indexes_api.h @@ -40,16 +40,18 @@ class CompositeIndexesApi : public ReindexerApi { ASSERT_TRUE(err.ok()) << err.what(); } - void addCompositeIndex(std::initializer_list indexes, CompositeIndexType type, const IndexOpts& opts) { + Error tryAddCompositeIndex(std::initializer_list indexes, CompositeIndexType type, const IndexOpts& opts) { reindexer::IndexDef indexDeclr; indexDeclr.name_ = getCompositeIndexName(indexes); indexDeclr.indexType_ = indexTypeToName(type); indexDeclr.fieldType_ = "composite"; indexDeclr.opts_ = opts; indexDeclr.jsonPaths_ = reindexer::JsonPaths(indexes); - Error err = rt.reindexer->AddIndex(default_namespace, indexDeclr); - EXPECT_TRUE(err.ok()) << err.what(); - err = rt.reindexer->Commit(default_namespace); + return rt.reindexer->AddIndex(default_namespace, indexDeclr); + } + + void addCompositeIndex(std::initializer_list indexes, CompositeIndexType type, const IndexOpts& opts) { + Error err = tryAddCompositeIndex(std::move(indexes), type, opts); EXPECT_TRUE(err.ok()) << err.what(); } diff --git a/cpp_src/gtests/tests/unit/composite_indexes_test.cc b/cpp_src/gtests/tests/unit/composite_indexes_test.cc index 97ae50578..b6ab2b70c 100644 --- a/cpp_src/gtests/tests/unit/composite_indexes_test.cc +++ b/cpp_src/gtests/tests/unit/composite_indexes_test.cc @@ -237,3 +237,51 @@ TEST_F(CompositeIndexesApi, SelectsBySubIndexes) { EXPECT_EQ(item[kFieldNamePages].As(), pagesValue) << c.name; } } + +TEST_F(CompositeIndexesApi, CompositeOverCompositeTest) { + constexpr char kExpectedErrorPattern[] = "Cannot create composite index '%s' over the other composite '%s'"; + constexpr size_t stepSize = 10; + size_t from = 0, to = stepSize; + auto addData = [this, &from, &to] { + fillNamespace(from, to); + from += stepSize; + to += stepSize; + }; + auto checkError = [this, &addData, &kExpectedErrorPattern](std::initializer_list compositeFields, + const std::string& singleField, CompositeIndexType type) { + auto compositeName = getCompositeIndexName(std::move(compositeFields)); + auto err = tryAddCompositeIndex({compositeName, singleField}, type, IndexOpts()); + EXPECT_EQ(err.code(), errParams) << compositeName; + EXPECT_EQ(err.what(), fmt::sprintf(kExpectedErrorPattern, getCompositeIndexName({compositeName, singleField}), compositeName)) + << compositeName; + addData(); + + err = tryAddCompositeIndex({singleField, compositeName}, type, IndexOpts()); + EXPECT_EQ(err.code(), errParams) << compositeName; + EXPECT_EQ(err.what(), fmt::sprintf(kExpectedErrorPattern, getCompositeIndexName({singleField, compositeName}), compositeName)) + << compositeName; + addData(); + }; + + addCompositeIndex({kFieldNameBookid, kFieldNameBookid2}, CompositeIndexHash, IndexOpts().PK()); + addData(); + + const std::string kNewIdxName = "new_idx"; + auto err = rt.reindexer->AddIndex(default_namespace, {kNewIdxName, {kNewIdxName}, "-", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + addData(); + + addCompositeIndex({kFieldNameTitle, kNewIdxName}, CompositeIndexHash, IndexOpts()); + addData(); + + checkError({kFieldNameBookid, kFieldNameBookid2}, kNewIdxName, CompositeIndexBTree); + checkError({kFieldNameTitle, kNewIdxName}, kFieldNamePrice, CompositeIndexBTree); + checkError({kFieldNameBookid, kFieldNameBookid2}, kFieldNamePrice, CompositeIndexHash); + + const auto kComposite1 = getCompositeIndexName({kFieldNameBookid, kFieldNameBookid2}); + const auto kComposite2 = getCompositeIndexName({kFieldNameTitle, kNewIdxName}); + err = tryAddCompositeIndex({kComposite1, kComposite2}, CompositeIndexHash, IndexOpts()); + EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.what(), fmt::sprintf(kExpectedErrorPattern, getCompositeIndexName({kComposite1, kComposite2}), kComposite1)); + addData(); +} diff --git a/cpp_src/gtests/tests/unit/ft/ft_generic.cc b/cpp_src/gtests/tests/unit/ft/ft_generic.cc index 7ff1be617..ac58f97de 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_generic.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_generic.cc @@ -583,7 +583,7 @@ TEST_P(FTGenericApi, Unique) { std::vector data; std::set check; std::set checks; - reindexer::logInstallWriter([](int, char*) { /*std::cout << buf << std::endl;*/ }); + reindexer::logInstallWriter([](int, char*) { /*std::cout << buf << std::endl;*/ }, reindexer::LoggerPolicy::WithLocks); for (int i = 0; i < 1000; ++i) { bool inserted = false; diff --git a/cpp_src/gtests/tests/unit/join_test.cc b/cpp_src/gtests/tests/unit/join_test.cc index e61e370d0..35f4f46f1 100644 --- a/cpp_src/gtests/tests/unit/join_test.cc +++ b/cpp_src/gtests/tests/unit/join_test.cc @@ -600,6 +600,37 @@ TEST_F(JoinSelectsApi, TestMergeWithJoins) { } } +// Check JOINs nested into the other JOINs (expecting errors) +TEST_F(JoinSelectsApi, TestNestedJoinsError) { + constexpr char sqlPattern[] = + R"(select * from books_namespace %s (select * from authors_namespace %s (select * from books_namespace) on authors_namespace.authorid = books_namespace.authorid_fk) on authors_namespace.authorid = books_namespace.authorid_fk)"; + auto joinTypes = {"inner join", "join", "left join"}; + for (auto& firstJoin : joinTypes) { + for (auto& secondJoin : joinTypes) { + auto sql = fmt::sprintf(sqlPattern, firstJoin, secondJoin); + ValidateQueryError(sql, errParams, "JOINs nested into the other JOINs are not supported"); + } + } +} + +// Check MERGEs nested into the JOINs (expecting errors) +TEST_F(JoinSelectsApi, TestNestedMergesInJoinsError) { + constexpr char sqlPattern[] = + R"(select * from books_namespace %s (select * from authors_namespace merge (select * from books_namespace)) on authors_namespace.authorid = books_namespace.authorid_fk)"; + auto joinTypes = {"inner join", "join", "left join"}; + for (auto& join : joinTypes) { + auto sql = fmt::sprintf(sqlPattern, join); + ValidateQueryError(sql, errParams, "MERGEs nested into the JOINs are not supported"); + } +} + +// Check MERGEs nested into the MERGEs (expecting errors) +TEST_F(JoinSelectsApi, TestNestedMergesInMergesError) { + constexpr char sql[] = + R"(select * from books_namespace merge (select * from authors_namespace merge (select * from books_namespace)))"; + ValidateQueryError(sql, errParams, "MERGEs nested into the MERGEs are not supported"); +} + TEST_F(JoinOnConditionsApi, TestGeneralConditions) { const std::string sqlTemplate = R"(select * from books_namespace inner join books_namespace on (books_namespace.authorid_fk = books_namespace.authorid_fk and books_namespace.pages %s books_namespace.pages);)"; diff --git a/cpp_src/gtests/tests/unit/namespace_test.cc b/cpp_src/gtests/tests/unit/namespace_test.cc index 911099d8c..4de15ddae 100644 --- a/cpp_src/gtests/tests/unit/namespace_test.cc +++ b/cpp_src/gtests/tests/unit/namespace_test.cc @@ -378,7 +378,7 @@ TEST_F(NsApi, QueryperfstatsNsDummyTest) { } } -void checkIfItemJSONValid(QueryResults::Iterator &it, bool print = false) { +static void checkIfItemJSONValid(QueryResults::Iterator &it, bool print = false) { reindexer::WrSerializer wrser; Error err = it.GetJSON(wrser, false); ASSERT_TRUE(err.ok()) << err.what(); @@ -608,8 +608,7 @@ TEST_F(NsApi, TestUpdateIndexedArrayField2) { QueryResults qr; VariantArray value; value.emplace_back(static_cast(77)); - value.MarkArray(); - Query q{Query(default_namespace).Where(idIdxName, CondEq, static_cast(1000)).Set(indexedArrayField, std::move(value))}; + Query q{Query(default_namespace).Where(idIdxName, CondEq, static_cast(1000)).Set(indexedArrayField, std::move(value.MarkArray()))}; Error err = rt.reindexer->Update(q, qr); ASSERT_TRUE(err.ok()) << err.what(); ASSERT_TRUE(qr.Count() == 1) << qr.Count(); @@ -1005,6 +1004,115 @@ TEST_F(NsApi, ExtendArrayWithExpressions) { } } +// Check if it's possible to use append operation with empty arrays and null fields +TEST_F(NsApi, ExtendEmptyArrayWithExpressions) { + const std::string kEmptyArraysNs = "empty_arrays_ns"; + CreateEmptyArraysNamespace(kEmptyArraysNs); + const Query kBaseQuery = Query(kEmptyArraysNs).Where("id", CondSet, {100, 105, 189, 113, 153}); + + auto ValidateResults = [this, &kBaseQuery, &kEmptyArraysNs](const QueryResults &qr, std::string_view pattern, std::string_view field, + const VariantArray &expectedValues, std::string_view description) { + const std::string fullDescription = "Description: " + std::string(description) + ";\n"; + // Check initial result + ASSERT_EQ(qr.Count(), 5) << fullDescription; + std::vector initialResults; + initialResults.reserve(qr.Count()); + for (auto it : qr) { + Item item = it.GetItem(false); + checkIfItemJSONValid(it); + const auto json = item.GetJSON(); + ASSERT_NE(json.find(pattern), std::string::npos) << fullDescription << "JSON: " << json << ";\npattern: " << pattern; + initialResults.emplace_back(json); + const VariantArray values = item[field]; + ASSERT_EQ(values.size(), expectedValues.size()) << fullDescription; + ASSERT_EQ(values.IsArrayValue(), expectedValues.IsArrayValue()) << fullDescription; + for (size_t i = 0; i < values.size(); ++i) { + ASSERT_TRUE(values[i].Type().IsSame(expectedValues[i].Type())) + << fullDescription << values[i].Type().Name() << "!=" << expectedValues[i].Type().Name(); + ASSERT_EQ(values[i], expectedValues[i]) << fullDescription; + } + } + // Check select results + QueryResults qrSelect; + const Query q = expectedValues.size() ? Query(kEmptyArraysNs).Where(std::string(field), CondAllSet, expectedValues) : kBaseQuery; + auto err = rt.reindexer->Select(q, qrSelect); + ASSERT_TRUE(err.ok()) << fullDescription << err.what(); + ASSERT_EQ(qrSelect.Count(), qr.Count()) << fullDescription; + unsigned i = 0; + for (auto it : qrSelect) { + Item item = it.GetItem(false); + checkIfItemJSONValid(it); + const auto json = item.GetJSON(); + ASSERT_EQ(json, initialResults[i++]) << fullDescription; + const VariantArray values = item[field]; + ASSERT_EQ(values.size(), expectedValues.size()) << fullDescription; + ASSERT_EQ(values.IsArrayValue(), expectedValues.IsArrayValue()) << fullDescription; + for (size_t j = 0; j < values.size(); ++j) { + ASSERT_TRUE(values[j].Type().IsSame(expectedValues[j].Type())) + << fullDescription << values[j].Type().Name() << "!=" << expectedValues[j].Type().Name(); + ASSERT_EQ(values[j], expectedValues[j]) << fullDescription; + } + } + }; + + { + const auto description = "append value to the empty indexed array"; + const Query query = Query(kBaseQuery).Set("indexed_array_field", Variant("indexed_array_field || [99, 99, 99]"), true); + QueryResults qr; + Error err = rt.reindexer->Update(query, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ValidateResults(qr, R"("indexed_array_field":[99,99,99],"non_indexed_array_field":[])", "indexed_array_field", + VariantArray{Variant(99), Variant(99), Variant(99)}, description); + } + { + const auto description = "append empty array to the indexed array"; + const Query query = Query(kBaseQuery).Set("indexed_array_field", Variant("indexed_array_field || []"), true); + QueryResults qr; + Error err = rt.reindexer->Update(query, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ValidateResults(qr, R"("indexed_array_field":[99,99,99],"non_indexed_array_field":[])", "indexed_array_field", + VariantArray{Variant(99), Variant(99), Variant(99)}, description); + } + { + const auto description = "append value to the empty non-indexed array"; + const Query query = Query(kBaseQuery).Set("non_indexed_array_field", Variant("non_indexed_array_field || [88, 88]"), true); + QueryResults qr; + Error err = rt.reindexer->Update(query, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ValidateResults(qr, R"("indexed_array_field":[99,99,99],"non_indexed_array_field":[88,88])", "non_indexed_array_field", + VariantArray{Variant(int64_t(88)), Variant(int64_t(88))}, description); + } + { + const auto description = "append empty array to the non-indexed array"; + const Query query = Query(kBaseQuery).Set("non_indexed_array_field", Variant("non_indexed_array_field || []"), true); + QueryResults qr; + Error err = rt.reindexer->Update(query, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ValidateResults(qr, R"("indexed_array_field":[99,99,99],"non_indexed_array_field":[88,88])", "non_indexed_array_field", + VariantArray{Variant(int64_t(88)), Variant(int64_t(88))}, description); + } + { + const auto description = "append empty array to the non-existing field"; + const Query query = Query(kBaseQuery).Set("non_existing_field", Variant("non_existing_field || []"), true); + QueryResults qr; + Error err = rt.reindexer->Update(query, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ValidateResults(qr, R"("indexed_array_field":[99,99,99],"non_indexed_array_field":[88,88],"non_existing_field":[])", + "non_existing_field", VariantArray().MarkArray(), description); + } + + { + const auto description = "append non-empty array to the non-existing field"; + const Query query = Query(kBaseQuery).Set("non_existing_field1", Variant("non_existing_field1 || [546]"), true); + QueryResults qr; + Error err = rt.reindexer->Update(query, qr); + ASSERT_TRUE(err.ok()) << err.what(); + ValidateResults( + qr, R"("indexed_array_field":[99,99,99],"non_indexed_array_field":[88,88],"non_existing_field":[],"non_existing_field1":[546])", + "non_existing_field1", VariantArray{Variant(int64_t(546))}.MarkArray(), description); + } +} + TEST_F(NsApi, UpdateObjectsArray) { // 1. Define NS // 2. Fill NS @@ -1081,29 +1189,31 @@ TEST_F(NsApi, UpdateObjectsArray4) { constexpr char kIndexName[] = "objects.array.field"; const Query kBaseQuery = Query(default_namespace).Where("id", CondSet, {1199, 1201, 1203, 1210, 1240}); - auto ValidateResults = [this, &kBaseQuery](const QueryResults &qr, std::string_view pattern, std::string_view indexTypeMsg) { + auto ValidateResults = [this, &kBaseQuery](const QueryResults &qr, std::string_view pattern, std::string_view indexType, + std::string_view description) { + const std::string fullDescription = fmt::sprintf("Description: %s; %s;\n", description, indexType); // Check initial result - ASSERT_GT(qr.Count(), 0) << indexTypeMsg; + ASSERT_EQ(qr.Count(), 5) << fullDescription; std::vector initialResults; initialResults.reserve(qr.Count()); for (auto it : qr) { Item item = it.GetItem(false); checkIfItemJSONValid(it); const auto json = item.GetJSON(); - ASSERT_NE(json.find(pattern), std::string::npos) << indexTypeMsg << ";\nJSON: " << json << ";\npattern: " << pattern; + ASSERT_NE(json.find(pattern), std::string::npos) << fullDescription << "JSON: " << json << ";\npattern: " << pattern; initialResults.emplace_back(json); } - // Checl select results + // Check select results QueryResults qrSelect; auto err = rt.reindexer->Select(kBaseQuery, qrSelect); - ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); - ASSERT_EQ(qrSelect.Count(), qr.Count()) << indexTypeMsg; + ASSERT_TRUE(err.ok()) << fullDescription << err.what(); + ASSERT_EQ(qrSelect.Count(), qr.Count()) << fullDescription; unsigned i = 0; for (auto it : qrSelect) { Item item = it.GetItem(false); checkIfItemJSONValid(it); const auto json = item.GetJSON(); - ASSERT_EQ(json, initialResults[i++]) << indexTypeMsg; + ASSERT_EQ(json, initialResults[i++]) << fullDescription; } }; @@ -1123,136 +1233,109 @@ TEST_F(NsApi, UpdateObjectsArray4) { const std::string indexTypeMsg = fmt::sprintf("Index type is '%s'", index); { - TestCout() << "Update array field, nested into objects array with explicit index (1 element). Index type: '" << index << "'" - << std::endl; + const auto description = "Update array field, nested into objects array with explicit index (1 element)"; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[0].field[4]", {777}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":[9,8,7,6,777]},{"field":11},{"field":[4,3,2,1,0]},{"field":[99]}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array with explicit index (1 element, different position). Index type: '" - << index << "'" << std::endl; + const auto description = "Update array field, nested into objects array with explicit index (1 element, different position)"; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[2].field[3]", {8387}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":[9,8,7,6,777]},{"field":11},{"field":[4,3,2,8387,0]},{"field":[99]}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index with scalar type. Index type: '" << index - << "'" << std::endl; + const auto description = "Update array field, nested into objects array without explicit index with scalar type"; // Make sure, that internal field's type ('scalar') was not changed Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", {537}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":[9,8,7,6,777]},{"field":537},{"field":[4,3,2,8387,0]},{"field":[99]}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update scalar field, nested into objects array with explicit index with array type. Index type: '" << index - << "'" << std::endl; + const auto description = "Update scalar field, nested into objects array with explicit index with array type"; // Make sure, that internal field's type ('array') was not changed Query updateQuery = Query(kBaseQuery).Set("objects[0].array[3].field[0]", {999}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":[9,8,7,6,777]},{"field":537},{"field":[4,3,2,8387,0]},{"field":[999]}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Change field type from array[1] to " - "scalar. Index type: '" - << index << "'" << std::endl; + const auto description = + "Update array field, nested into objects array without explicit index. Change field type from array[1] to scalar"; // Make sure, that internal field's type (array of 1 element) was changed to scalar Query updateQuery = Query(kBaseQuery).Set("objects[0].array[3].field", {837}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":[9,8,7,6,777]},{"field":537},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Change field type from array[4] to " - "scalar. Index type: '" - << index << "'" << std::endl; + const auto description = + "Update array field, nested into objects array without explicit index. Change field type from array[4] to scalar"; // Make sure, that internal field's type (array of 4 elements) was changed to scalar Query updateQuery = Query(kBaseQuery).Set("objects[0].array[0].field", {2345}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":537},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Change field type from scalar to " - "array[1]. Index type: '" - << index << "'" << std::endl; + const auto description = + "Update array field, nested into objects array without explicit index. Change field type from scalar to array[1]"; // Make sure, that internal field's type ('scalar') was changed to array - VariantArray v{Variant{1847}}; - v.MarkArray(); - Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", v, false); + Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", VariantArray{Variant{1847}}.MarkArray(), false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[1847]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Increase array size. Index type: '" - << index << "'" << std::endl; - VariantArray v{Variant{115}, Variant{1000}, Variant{501}}; - Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", v, false); + const auto description = "Update array field, nested into objects array without explicit index. Increase array size"; + Query updateQuery = + Query(kBaseQuery).Set("objects[0].array[1].field", VariantArray{Variant{115}, Variant{1000}, Variant{501}}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[115,1000,501]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Reduce array size (to multiple elements). " - "Index type: '" - << index << "'" << std::endl; - VariantArray v{Variant{100}, Variant{999}}; - Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", v, false); + const auto description = + "Update array field, nested into objects array without explicit index. Reduce array size (to multiple elements)"; + Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", VariantArray{Variant{100}, Variant{999}}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[100,999]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Reduce array size (to single element). " - "Index type: '" - << index << "'" << std::endl; - VariantArray v{Variant{150}}; - v.MarkArray(); - Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", v, false); + const auto description = + "Update array field, nested into objects array without explicit index. Reduce array size (to single element)"; + Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", VariantArray{Variant{150}}.MarkArray(), false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[150]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Attempt to set array-value(1 element) by explicit index. Index type: '" << index << "'" << std::endl; - VariantArray v{Variant{199}}; - v.MarkArray(); - Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field[0]", v, false); + const auto description = "Attempt to set array-value(1 element) by explicit index"; + Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field[0]", VariantArray{Variant{199}}.MarkArray(), false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_EQ(err.code(), errParams) << indexTypeMsg << err.what(); @@ -1262,11 +1345,10 @@ TEST_F(NsApi, UpdateObjectsArray4) { ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); // Make sure, that item was not changed ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[150]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Attempt to set array-value(multiple elements) by explicit index. Index type: '" << index << "'" << std::endl; + const auto description = "Attempt to set array-value(multiple elements) by explicit index"; VariantArray v{Variant{199}, Variant{200}, Variant{300}}; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field[0]", v, false); QueryResults qr; @@ -1278,11 +1360,10 @@ TEST_F(NsApi, UpdateObjectsArray4) { ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); // Make sure, that item was not changed ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[150]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Attempt to set array-value(1 element) by *-index. Index type: '" << index << "'" << std::endl; + const auto description = "Attempt to set array-value(1 element) by *-index"; VariantArray v{Variant{199}, Variant{200}, Variant{300}}; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field[*]", v, false); QueryResults qr; @@ -1294,11 +1375,10 @@ TEST_F(NsApi, UpdateObjectsArray4) { ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); // Make sure, that item was not changed ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[150]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Attempt to set array-value(multiple elements) by *-index. Index type: '" << index << "'" << std::endl; + const auto description = "Attempt to set array-value(multiple elements) by *-index"; VariantArray v{Variant{199}, Variant{200}, Variant{300}}; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field[*]", v, false); QueryResults qr; @@ -1310,21 +1390,19 @@ TEST_F(NsApi, UpdateObjectsArray4) { ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); // Make sure, that item was not changed ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[150]},{"field":[4,3,2,8387,0]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array with *-index. Index type: '" << index << "'" << std::endl; + const auto description = "Update array field, nested into objects array with *-index"; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[2].field[*]", {199}, false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[150]},{"field":[199,199,199,199,199]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Attempt to update scalar value by *-index. Index type: '" << index << "'" << std::endl; + const auto description = "Attempt to update scalar value by *-index"; VariantArray v{Variant{199}, Variant{200}, Variant{300}}; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[0].field[*]", v, false); QueryResults qr; @@ -1336,25 +1414,19 @@ TEST_F(NsApi, UpdateObjectsArray4) { ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); // Make sure, that item was not changed ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[150]},{"field":[199,199,199,199,199]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Reduce array size to 0. Index type: '" - << index << "'" << std::endl; - VariantArray v; - v.MarkArray(); - Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", v, false); + const auto description = "Update array field, nested into objects array without explicit index. Reduce array size to 0"; + Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", VariantArray().MarkArray(), false); QueryResults qr; err = rt.reindexer->Update(updateQuery, qr); ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults(qr, R"("objects":[{"array":[{"field":2345},{"field":[]},{"field":[199,199,199,199,199]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); } - { - TestCout() << "Update array field, nested into objects array without explicit index. Increase array size from 0. Index type: '" - << index << "'" << std::endl; + const auto description = "Update array field, nested into objects array without explicit index. Increase array size from 0"; VariantArray v{Variant{11199}, Variant{11200}, Variant{11300}}; Query updateQuery = Query(kBaseQuery).Set("objects[0].array[1].field", v, false); QueryResults qr; @@ -1362,9 +1434,60 @@ TEST_F(NsApi, UpdateObjectsArray4) { ASSERT_TRUE(err.ok()) << indexTypeMsg << err.what(); ValidateResults( qr, R"("objects":[{"array":[{"field":2345},{"field":[11199,11200,11300]},{"field":[199,199,199,199,199]},{"field":837}]}])", - indexTypeMsg); + indexTypeMsg, description); + } + } +} + +TEST_F(NsApi, UpdateWithObjectAndFieldsDuplication) { + Error err = rt.reindexer->OpenNamespace(default_namespace); + ASSERT_TRUE(err.ok()) << err.what(); + + err = rt.reindexer->AddIndex(default_namespace, {"id", "hash", "int", IndexOpts().PK()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"nested", reindexer::JsonPaths{"n.idv"}, "hash", "string", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + + std::vector items = {R"({"id":0,"data":"data0","n":{"idv":"index_str_1","dat":"data1"}})", + R"({"id":5,"data":"data5","n":{"idv":"index_str_3","dat":"data2"}})"}; + + AddItemFromJSON(default_namespace, items[0]); + AddItemFromJSON(default_namespace, items[1]); + + { + QueryResults qr; + err = rt.reindexer->Update(Query(default_namespace) + .SetObject("n", R"({"idv":"index_str_3_modified","idv":"index_str_5_modified","dat":"data2_mod"})") + .Where("id", CondEq, 5), + qr); + ASSERT_EQ(err.code(), errLogic) << err.what(); + } + { + // Check all the items + QueryResults qr; + err = rt.reindexer->Select(Query(default_namespace).Sort("id", false), qr); + ASSERT_EQ(qr.Count(), 2); + unsigned i = 0; + for (auto it : qr) { + ASSERT_EQ(it.GetItem().GetJSON(), items[i]) << i; + ++i; } } + { + // Check old indexed value (have to exists) + QueryResults qr; + err = rt.reindexer->Select(Query(default_namespace).Where("nested", CondEq, std::string("index_str_3")), qr); + ASSERT_EQ(qr.Count(), 1); + ASSERT_EQ(qr.begin().GetItem().GetJSON(), items[1]); + } + { + // Check new indexed values (have to not exist) + QueryResults qr; + err = rt.reindexer->Select( + Query(default_namespace).Where("nested", CondSet, {std::string("index_str_3_modified"), std::string("index_str_5_modified")}), + qr); + ASSERT_EQ(qr.Count(), 0); + } } TEST_F(NsApi, UpdateOutOfBoundsArrayField) { @@ -2060,7 +2183,7 @@ TEST_F(NsApi, TestUpdateFieldWithExpressions) { } } -void checkQueryDsl(const Query &src) { +static void checkQueryDsl(const Query &src) { Query dst; const std::string dsl = src.GetJSON(); Error err = dst.FromJSON(dsl); @@ -2150,7 +2273,7 @@ TEST_F(NsApi, TestModifyQueriesSqlEncoder) { checkQueryDsl(q7); } -void generateObject(reindexer::JsonBuilder &builder, const std::string &prefix, ReindexerApi *rtapi) { +static void generateObject(reindexer::JsonBuilder &builder, const std::string &prefix, ReindexerApi *rtapi) { builder.Put(prefix + "ID", rand() % 1000); builder.Put(prefix + "Name", rtapi->RandString()); builder.Put(prefix + "Rating", rtapi->RandString()); diff --git a/cpp_src/gtests/tests/unit/queries_test.cc b/cpp_src/gtests/tests/unit/queries_test.cc index d272ac8d7..5a9ac6eb7 100644 --- a/cpp_src/gtests/tests/unit/queries_test.cc +++ b/cpp_src/gtests/tests/unit/queries_test.cc @@ -26,6 +26,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { CheckDistinctQueries(); CheckGeomQueries(); CheckMergeQueriesWithLimit(); + CheckMergeQueriesWithAggregation(); int itemsCount = 0; auto& items = insertedItems_[default_namespace]; diff --git a/cpp_src/gtests/tests/unit/string_function_test.cc b/cpp_src/gtests/tests/unit/string_function_test.cc index 19fe1b402..441a1e29c 100644 --- a/cpp_src/gtests/tests/unit/string_function_test.cc +++ b/cpp_src/gtests/tests/unit/string_function_test.cc @@ -105,6 +105,37 @@ TEST(StringFunctions, IsLikeSqlPattern) { } } +// test to check +// 1. equality of character length in bytes for uppercase and lowercase letters +// 2. character length equality after the substitution function 'check_for_replacement' + +TEST(StringFunctions, ToLowerUTF8ByteLen) { + for (wchar_t a = 0; a < UINT16_MAX; ++a) { + auto utf8ByteSize = [](wchar_t a) { + std::string symUtf8; + std::wstring symIn; + symIn += a; + reindexer::utf16_to_utf8(symIn, symUtf8); + return symUtf8.size(); + }; + ASSERT_EQ(utf8ByteSize(a), utf8ByteSize(reindexer::ToLower(a))); + { + wchar_t replaceChar = a; + reindexer::check_for_replacement(replaceChar); + if (replaceChar != a) { + ASSERT_EQ(utf8ByteSize(a), utf8ByteSize(replaceChar)); + } + } + { + uint32_t replaceChar = a; + reindexer::check_for_replacement(replaceChar); + if (replaceChar != uint32_t(a)) { + ASSERT_EQ(utf8ByteSize(a), utf8ByteSize(replaceChar)); + } + } + } +} + // Make sure 'Like' operator does not work with FT indexes TEST_F(ReindexerApi, LikeWithFullTextIndex) { // Define structure of the Namespace, where one of @@ -113,7 +144,7 @@ TEST_F(ReindexerApi, LikeWithFullTextIndex) { ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->AddIndex(default_namespace, {"id", {"id"}, "hash", "int", IndexOpts().PK()}); ASSERT_TRUE(err.ok()) << err.what(); - err = rt.reindexer->AddIndex(default_namespace, {"name", {"name", "text"}, "text", "string", IndexOpts()}); + err = rt.reindexer->AddIndex(default_namespace, {"name", {"name"}, "text", "string", IndexOpts()}); ASSERT_TRUE(err.ok()) << err.what(); // Insert 100 items to newly created Namespace diff --git a/cpp_src/gtests/tests/unit/value_by_json_path.cc b/cpp_src/gtests/tests/unit/value_by_json_path.cc index 5be704997..5d2797dbc 100644 --- a/cpp_src/gtests/tests/unit/value_by_json_path.cc +++ b/cpp_src/gtests/tests/unit/value_by_json_path.cc @@ -251,34 +251,3 @@ TEST_F(ReindexerApi, NumericSearchForNonIndexedField) { ASSERT_TRUE(value.Type().Is()); } } - -TEST_F(ReindexerApi, InsertWithSeveralJsonPaths) { - // Define namespace structure with an indexed field that has 3 json paths - Error err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); - ASSERT_TRUE(err.ok()) << err.what(); - - err = rt.reindexer->AddIndex(default_namespace, {"id", "hash", "int", IndexOpts().PK()}); - ASSERT_TRUE(err.ok()) << err.what(); - - err = rt.reindexer->AddIndex(default_namespace, {"name", {"name", "text", "description"}, "hash", "string", IndexOpts(), 0}); - ASSERT_TRUE(err.ok()) << err.what(); - - // Build an item, which includes (and sets) all the json-paths for the field 'name' - reindexer::WrSerializer wrser; - reindexer::JsonBuilder jsonBuilder(wrser, reindexer::ObjType::TypeObject); - jsonBuilder.Put("id", int(1)); - jsonBuilder.Put("name", "first"); - jsonBuilder.Put("text", "second"); - jsonBuilder.Put("description", "third"); - jsonBuilder.End(); - - // Insert the item - Item item = rt.reindexer->NewItem(default_namespace); - ASSERT_TRUE(item.Status().ok()) << item.Status().what(); - err = item.FromJSON(wrser.Slice()); - ASSERT_TRUE(err.ok()) << err.what(); - err = rt.reindexer->Insert(default_namespace, item); - - // Make sure it returned an error of type 'errParams' - ASSERT_TRUE(!err.ok() && err.code() == errParams); -} diff --git a/cpp_src/net/connection.cc b/cpp_src/net/connection.cc index a140c69a3..6187fba4a 100644 --- a/cpp_src/net/connection.cc +++ b/cpp_src/net/connection.cc @@ -160,7 +160,19 @@ typename Connection::ReadResT Connection::read_cb() { return ReadResT::Default; } template -void Connection::timeout_cb(ev::periodic & /*watcher*/, int /*time*/) { +void Connection::timeout_cb(ev::periodic &watcher, int /*time*/) { + if (sock_.has_pending_data()) { + fprintf(stdout, + "Connection got idle timeout, but socket has pending data. Do not dropping the connection\nThis probably means, that " + "there are some very long queries in some of the connections, which may affect the other connections. Consider to use " + "dedicated threads for them\n"); + if (!watcher.has_period()) { + watcher.start(watcher.last_delay()); + } + return; + } + + fprintf(stderr, "Dropping RPC-connection on the idle timeout\n"); closeConn(); } template diff --git a/cpp_src/net/cproto/dispatcher.h b/cpp_src/net/cproto/dispatcher.h index 35fb4abec..703fe9dad 100644 --- a/cpp_src/net/cproto/dispatcher.h +++ b/cpp_src/net/cproto/dispatcher.h @@ -34,7 +34,7 @@ struct ClientData { struct Context; struct IRPCCall { - void (*Get)(IRPCCall *, CmdCode &, Args &); + void (*Get)(IRPCCall *, CmdCode &, std::string_view &nsName, Args &); intrusive_ptr> data_; }; diff --git a/cpp_src/net/cproto/serverconnection.cc b/cpp_src/net/cproto/serverconnection.cc index 950b5e2f6..d6cd24ad9 100644 --- a/cpp_src/net/cproto/serverconnection.cc +++ b/cpp_src/net/cproto/serverconnection.cc @@ -265,48 +265,41 @@ void ServerConnection::responceRPC(Context &ctx, const Error &status, const Args } void ServerConnection::CallRPC(const IRPCCall &call) { - std::unique_lock lck(updates_mtx_); + std::lock_guard lck(updates_mtx_); + updates_.emplace_back(call); + updatesSize_ += call.data_->size(); + if (updatesSize_ > maxUpdatesSize_) { updates_.clear(); Args args; IRPCCall curCall = call; CmdCode cmd; - curCall.Get(&curCall, cmd, args); - if (args.size() >= 3) { - std::string_view nsName(args[1]); - WrSerializer ser; - ser.PutVString(nsName); - intrusive_ptr> data; - data.reset(new intrusive_atomic_rc_wrapper(ser.DetachChunk())); - IRPCCall callLost = {[](IRPCCall *callLost, CmdCode &cmd, Args &args) { - Serializer ser(callLost->data_->data(), callLost->data_->size()); - auto nsName = ser.GetVString(); - cmd = kCmdUpdates; - args = {Arg(std::string(nsName.data(), nsName.size()))}; - }, - data}; - logPrintf(LogWarning, "Call updates lost clientAddr = %s updatesSize = %d", clientAddr_, updatesSize_); - - updates_.emplace_back(callLost); - } - - // order is important + std::string_view nsName; + curCall.Get(&curCall, cmd, nsName, args); + + WrSerializer ser; + ser.PutVString(nsName); + IRPCCall callLost = {[](IRPCCall *callLost, CmdCode &cmd, std::string_view &ns, Args &args) { + Serializer s(callLost->data_->data(), callLost->data_->size()); + cmd = kCmdUpdates; + args = {Arg(std::string(s.GetVString()))}; + ns = std::string_view(args[0]); + }, + make_intrusive>(ser.DetachChunk())}; + logPrintf(LogWarning, "Call updates lost clientAddr = %s updatesSize = %d", clientAddr_, updatesSize_); + + updatesSize_ = callLost.data_->size(); + updates_.emplace_back(std::move(callLost)); updateLostFlag_ = true; - updatesSize_ = 0; if (ConnectionST::stats_) { - auto stat = ConnectionST::stats_->get_stat(); - if (stat) { - stat->updates_lost++; + if (auto stat = ConnectionST::stats_->get_stat(); stat) { + stat->updates_lost.fetch_add(1, std::memory_order_relaxed); + stat->pended_updates.store(1, std::memory_order_relaxed); } } - } - updates_.emplace_back(call); - updatesSize_ += call.data_->size(); - - if (ConnectionST::stats_) { - auto stat = ConnectionST::stats_->get_stat(); - if (stat) { + } else if (ConnectionST::stats_) { + if (auto stat = ConnectionST::stats_->get_stat(); stat) { stat->pended_updates.store(updates_.size(), std::memory_order_relaxed); } } @@ -318,10 +311,15 @@ void ServerConnection::sendUpdates() { } std::vector updates; - updates_mtx_.lock(); - updates.swap(updates_); - updateLostFlag_ = false; - updates_mtx_.unlock(); + size_t updatesSizeCopy; + { + std::lock_guard lck(updates_mtx_); + updates.swap(updates_); + updatesSizeCopy = updatesSize_; + updatesSize_ = 0; + updateLostFlag_ = false; + } + if (updates.empty()) { return; } @@ -333,13 +331,13 @@ void ServerConnection::sendUpdates() { CmdCode cmd; WrSerializer ser(wrBuf_.get_chunk()); size_t cnt = 0; + size_t updatesSizeBuffered = 0; for (cnt = 0; cnt < updates.size() && ser.Len() < kMaxUpdatesBufSize; ++cnt) { if (updates[cnt].data_) { - if (!updateLostFlag_) { - updatesSize_ -= updates[cnt].data_->size(); - } + updatesSizeBuffered += updates[cnt].data_->size(); } - updates[cnt].Get(&updates[cnt], cmd, args); + [[maybe_unused]] std::string_view ns; + updates[cnt].Get(&updates[cnt], cmd, ns, args); packRPC(ser, ctx, Error(), args, enableSnappy_); } @@ -347,20 +345,46 @@ void ServerConnection::sendUpdates() { try { wrBuf_.write(ser.DetachChunk()); } catch (...) { + RPCCall callLost{kCmdUpdates, 0, {}, milliseconds(0)}; + cproto::Context ctxLost{"", &callLost, this, {{}, {}}, false}; + { + std::lock_guard lck(updates_mtx_); + updates_.clear(); + updatesSize_ = 0; + updateLostFlag_ = false; + + logPrintf(LogWarning, "Call updates lost clientAddr = %s (wrBuf error)", clientAddr_); + wrBuf_.clear(); + ser.Reset(); + packRPC(ser, ctxLost, Error(), {Arg(std::string(""))}, enableSnappy_); + len = ser.Len(); + wrBuf_.write(ser.DetachChunk()); + if (ConnectionST::stats_) { + ConnectionST::stats_->update_send_buf_size(wrBuf_.data_size()); + ConnectionST::stats_->update_pended_updates(0); + } + } + + if (dispatcher_.onResponse_) { + ctx.stat.sizeStat.respSizeBytes = len; + dispatcher_.onResponse_(ctxLost); + } + + callback(io_, ev::WRITE); return; } if (cnt != updates.size()) { - std::unique_lock lck(updates_mtx_); + std::lock_guard lck(updates_mtx_); if (!updateLostFlag_) { updates_.insert(updates_.begin(), updates.begin() + cnt, updates.end()); + updatesSize_ += updatesSizeCopy - updatesSizeBuffered; } if (ConnectionST::stats_) stats_->update_pended_updates(updates.size()); } else if (ConnectionST::stats_) { - auto stat = ConnectionST::stats_->get_stat(); - if (stat) { - std::unique_lock lck(updates_mtx_); + if (auto stat = ConnectionST::stats_->get_stat(); stat) { + std::lock_guard lck(updates_mtx_); stat->pended_updates.store(updates_.size(), std::memory_order_relaxed); } } diff --git a/cpp_src/net/cproto/serverconnection.h b/cpp_src/net/cproto/serverconnection.h index f7d817f4d..c2d155e6b 100644 --- a/cpp_src/net/cproto/serverconnection.h +++ b/cpp_src/net/cproto/serverconnection.h @@ -70,8 +70,8 @@ class ServerConnection : public ConnectionST, public IServerConnection, public W RPCCall call_; std::vector updates_; - std::atomic updatesSize_; - std::atomic updateLostFlag_; + size_t updatesSize_; + bool updateLostFlag_; const size_t maxUpdatesSize_; std::mutex updates_mtx_; diff --git a/cpp_src/net/ev/ev.h b/cpp_src/net/ev/ev.h index 61ad8bfcc..d0e9608a8 100644 --- a/cpp_src/net/ev/ev.h +++ b/cpp_src/net/ev/ev.h @@ -310,6 +310,7 @@ class timer { void set(dynamic_loop &loop_) noexcept { loop.loop_ = &loop_; } void start(double t, double p = 0) { period_ = p; + t_ = t; loop.set(this, t); } void stop() { loop.stop(this); } @@ -322,6 +323,8 @@ class timer { void set(std::function func) noexcept { func_ = std::move(func); } bool is_active() const noexcept { return loop.is_active(this); } + double last_delay() const noexcept { return t_; } + bool has_period() const noexcept { return period_ > 0.00000001; } loop_ref loop; std::chrono::time_point deadline_; @@ -337,7 +340,7 @@ class timer { func(*this, tv); // Timer is deallocated after this call } else { func_(*this, tv); - if (period_ > 0.00000001) { + if (has_period()) { loop.set(this, period_); } } @@ -345,6 +348,7 @@ class timer { std::function func_ = nullptr; double period_ = 0; + double t_ = 0; bool in_coro_storage_ = false; }; diff --git a/cpp_src/net/socket.cc b/cpp_src/net/socket.cc index f2cf26083..a0a1bcf3d 100644 --- a/cpp_src/net/socket.cc +++ b/cpp_src/net/socket.cc @@ -206,6 +206,26 @@ int socket::set_nodelay() { return setsockopt(fd_, SOL_TCP, TCP_NODELAY, reinterpret_cast(&flag), sizeof(flag)); } +bool socket::has_pending_data() const noexcept { + if (!valid()) { + return false; + } +#ifndef _WIN32 + int count; + if (ioctl(fd(), FIONREAD, &count) < 0) { + perror("ioctl(FIONREAD)"); + return false; + } +#else + u_long count = -1; + if (ioctlsocket(fd(), FIONREAD, &count) < 0) { + perror("ioctlsocket(FIONREAD)"); + return false; + } +#endif + return count > 0; +} + int socket::last_error() { #ifndef _WIN32 return errno; diff --git a/cpp_src/net/socket.h b/cpp_src/net/socket.h index 8f4b06a08..9557dd26c 100644 --- a/cpp_src/net/socket.h +++ b/cpp_src/net/socket.h @@ -26,8 +26,9 @@ class socket { int set_nonblock(); int set_nodelay(); - int fd() const { return fd_; } - bool valid() const { return fd_ >= 0; } + int fd() const noexcept { return fd_; } + bool valid() const noexcept { return fd_ >= 0; } + bool has_pending_data() const noexcept; static int last_error(); static bool would_block(int error); diff --git a/cpp_src/replicator/replicator.cc b/cpp_src/replicator/replicator.cc index 9d9215349..4717903e3 100644 --- a/cpp_src/replicator/replicator.cc +++ b/cpp_src/replicator/replicator.cc @@ -774,7 +774,7 @@ Error Replicator::applyWALRecord(LSNPair LSNs, std::string_view nsName, Namespac break; // Metadata updated case WalPutMeta: - slaveNs->PutMeta(std::string(rec.putMeta.key), rec.putMeta.value, NsContext(dummyCtx_)); + slaveNs->PutMeta(std::string(rec.putMeta.key), rec.putMeta.value, dummyCtx_); stat.updatedMeta++; break; // Update query @@ -784,16 +784,15 @@ Error Replicator::applyWALRecord(LSNPair LSNs, std::string_view nsName, Namespac QueryResults result; Query q; q.FromSQL(rec.data); - const auto nsCtx = NsContext(rdxContext); switch (q.type_) { case QueryDelete: - slaveNs->Delete(q, result, nsCtx); + slaveNs->Delete(q, result, rdxContext); break; case QueryUpdate: - slaveNs->Update(q, result, nsCtx); + slaveNs->Update(q, result, rdxContext); break; case QueryTruncate: - slaveNs->Truncate(nsCtx); + slaveNs->Truncate(rdxContext); break; case QuerySelect: break; @@ -864,22 +863,21 @@ Error Replicator::modifyItem(LSNPair LSNs, Namespace::Ptr &slaveNs, std::string_ if (err.ok()) { RdxContext rdxContext(true, LSNs); - auto nsCtx = NsContext(rdxContext); switch (modifyMode) { case ModeDelete: - slaveNs->Delete(item, nsCtx); + slaveNs->Delete(item, rdxContext); stat.deleted++; break; case ModeInsert: - slaveNs->Insert(item, nsCtx); + slaveNs->Insert(item, rdxContext); stat.updated++; break; case ModeUpsert: - slaveNs->Upsert(item, nsCtx); + slaveNs->Upsert(item, rdxContext); stat.updated++; break; case ModeUpdate: - slaveNs->Update(item, nsCtx); + slaveNs->Update(item, rdxContext); stat.updated++; break; default: @@ -946,7 +944,7 @@ Error Replicator::syncMetaForced(Namespace::Ptr &slaveNs, std::string_view nsNam continue; } try { - slaveNs->PutMeta(key, data, NsContext(dummyCtx_)); + slaveNs->PutMeta(key, data, dummyCtx_); } catch (const Error &e) { logPrintf(LogError, "[repl:%s]:%d Error set meta '%s': %s", slaveNs->GetName(dummyCtx_), config_.serverId, key, e.what()); } diff --git a/cpp_src/server/CMakeLists.txt b/cpp_src/server/CMakeLists.txt index b7fa6537d..aa4541461 100644 --- a/cpp_src/server/CMakeLists.txt +++ b/cpp_src/server/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.0) project(reindexer_server_library) set (SWAGGER_VERSION "2.x") -set (GH_FACE_VERSION "3.18.0") +set (GH_FACE_VERSION "3.19.0") set (GH_FACE_TAG "v${GH_FACE_VERSION}") set (TARGET reindexer_server_library) set (SERVER_LIB_DIR ${PROJECT_BINARY_DIR} PARENT_SCOPE) diff --git a/cpp_src/server/contrib/server.md b/cpp_src/server/contrib/server.md index 85a230596..7b092c81c 100644 --- a/cpp_src/server/contrib/server.md +++ b/cpp_src/server/contrib/server.md @@ -130,7 +130,7 @@ Reindexer is fast. ### Version information -*Version* : 3.18.0 +*Version* : 3.19.0 ### License information @@ -2138,15 +2138,44 @@ Query execution explainings |---|---|---| |**general_sort_us**
*optional*|Result sort time|integer| |**indexes_us**
*optional*|Indexes keys selection time|integer| +|**join_on_conditions**
*optional*|Describes Join ON conditions injections|< [join_on_conditions](#explaindef-join_on_conditions) > array| |**loop_us**
*optional*|Intersection loop time|integer| |**postprocess_us**
*optional*|Query post process time|integer| |**prepare_us**
*optional*|Query prepare and optimize time|integer| -|**selectors**
*optional*|Filter selectos, used to proccess query conditions|< [selectors](#explaindef-selectors) > array| +|**preselect_us**
*optional*|Query preselect processing time|integer| +|**selectors**
*optional*|Filter selectors, used to proccess query conditions|< [selectors](#explaindef-selectors) > array| |**sort_by_uncommitted_index**
*optional*|Optimization of sort by uncompleted index has been performed|boolean| |**sort_index**
*optional*|Index, which used for sort results|string| |**total_us**
*optional*|Total query execution time|integer| +**join_on_conditions** + +|Name|Description|Schema| +|---|---|---| +|**conditions**
*optional*|Individual conditions processing results|< [conditions](#explaindef-conditions) > array| +|**injected_condition**
*optional*|Injected condition. SQL-like string|string| +|**namespace**
*optional*|Joinable ns name|string| +|**on_condition**
*optional*|Original ON-conditions clause. SQL-like string|string| +|**reason**
*optional*|Optional{succeed==false}. Explains condition injection failure|string| +|**success**
*optional*|Result of injection attempt|boolean| +|**type**
*optional*|Values source: preselect values(by_value) or additional select(select)|string| + + +**conditions** + +|Name|Description|Schema| +|---|---|---| +|**agg_type**
*optional*|Optional. Aggregation type used in subquery|enum (min, max, distinct)| +|**condition**
*optional*|single condition from Join ON section. SQL-like string|string| +|**explain_select**
*optional*|Optional. Explain of Select subquery|[ExplainDef](#explaindef)| +|**new_condition**
*optional*|substituted injected condition. SQL-like string|string| +|**reason**
*optional*|Optional. Explains condition injection failure|string| +|**success**
*optional*|result of injection attempt|boolean| +|**total_time_us**
*optional*|total time elapsed from injection attempt start till the end of substitution or rejection|integer| +|**values_count**
*optional*|resulting size of query values set|integer| + + **selectors** |Name|Description|Schema| diff --git a/cpp_src/server/contrib/server.yml b/cpp_src/server/contrib/server.yml index 0b6b39ab3..d2906ac08 100644 --- a/cpp_src/server/contrib/server.yml +++ b/cpp_src/server/contrib/server.yml @@ -5,7 +5,7 @@ info: Reindexer's goal is to provide fast search with complex queries. The Reindexer is compact and fast. It has not heavy dependencies. Complete reindexer docker image with all libraries and web interface size is just 15MB. Reindexer is fast. - version: "3.18.0" + version: "3.19.0" title: "Reindexer REST API" license: name: "Apache 2.0" @@ -3077,6 +3077,9 @@ definitions: postprocess_us: type: integer description: "Query post process time" + preselect_us: + type: integer + description: "Query preselect processing time" prepare_us: type: integer description: "Query prepare and optimize time" @@ -3091,7 +3094,7 @@ definitions: description: "Optimization of sort by uncompleted index has been performed" selectors: type: array - description: "Filter selectos, used to proccess query conditions" + description: "Filter selectors, used to proccess query conditions" items: type: object properties: @@ -3127,6 +3130,68 @@ definitions: explain_select: description: "One of selects in joined namespace execution explainings" $ref: "#/definitions/ExplainDef" + join_on_conditions: + type: array + description: "Describes Join ON conditions injections" + items: + type: object + properties: + namespace: + type: string + description: "Joinable ns name" + on_condition: + type: string + description: "Original ON-conditions clause. SQL-like string" + total_time_us: + type: int + description: "Total amount of time spent on checking and substituting all conditions" + success: + type: boolean + description: "Result of injection attempt" + reason: + type: string + description: "Optional{succeed==false}. Explains condition injection failure" + type: + type: string + description: "Values source: preselect values(by_value) or additional select(select)" + injected_condition: + type: string + description: "Injected condition. SQL-like string" + conditions: + type: array + description: "Individual conditions processing results" + items: + type: object + properties: + condition: + type: string + description: single condition from Join ON section. SQL-like string + total_time_us: + type: integer + description: total time elapsed from injection attempt start till the end of substitution or rejection + explain_select: + description: Optional. Explain of Select subquery + $ref: "#/definitions/ExplainDef" + agg_type: + type: string + description: Optional. Aggregation type used in subquery + enum: + - "min" + - "max" + - "distinct" + success: + type: boolean + description: result of injection attempt + reason: + type: string + description: Optional. Explains condition injection failure + new_condition: + type: string + description: substituted injected condition. SQL-like string + values_count: + type: integer + description: resulting size of query values set + AggregationResDef: type: object diff --git a/cpp_src/server/pprof/pprof.cc b/cpp_src/server/pprof/pprof.cc index f059be6d7..f782f00d2 100644 --- a/cpp_src/server/pprof/pprof.cc +++ b/cpp_src/server/pprof/pprof.cc @@ -12,7 +12,7 @@ #include "tools/serializer.h" #include "tools/stringstools.h" -const std::string kProfileNamePrefix = "reindexer_server"; +static const std::string kProfileNamePrefix = "reindexer_server"; namespace reindexer_server { using namespace reindexer; diff --git a/cpp_src/server/rpcserver.cc b/cpp_src/server/rpcserver.cc index 10e8d3644..c0731aa5d 100644 --- a/cpp_src/server/rpcserver.cc +++ b/cpp_src/server/rpcserver.cc @@ -113,7 +113,7 @@ static RPCClientData *getClientDataUnsafe(cproto::Context &ctx) { return dynamic static RPCClientData *getClientDataSafe(cproto::Context &ctx) { auto ret = dynamic_cast(ctx.GetClientData()); - if (rx_unlikely(!ret)) std::abort(); // It has to be set by the middleware + if rx_unlikely (!ret) std::abort(); // It has to be set by the middleware return ret; } @@ -593,7 +593,8 @@ Error RPCServer::DeleteQuery(cproto::Context &ctx, p_string queryBin, std::optio if (flagsOpts) { flags = *flagsOpts; } - ResultFetchOpts opts{.flags = flags, .ptVersions = {}, .fetchOffset = 0, .fetchLimit = INT_MAX, .withAggregations = true}; + int32_t ptVersion = -1; + ResultFetchOpts opts{.flags = flags, .ptVersions = {&ptVersion, 1}, .fetchOffset = 0, .fetchLimit = INT_MAX, .withAggregations = true}; return sendResults(ctx, qres, RPCQrId(), opts); } @@ -625,7 +626,7 @@ Reindexer RPCServer::getDB(cproto::Context &ctx, UserRole role) { if (rx_likely(clientData)) { Reindexer *db = nullptr; auto status = clientData->auth.GetDB(role, &db); - if (rx_unlikely(!status.ok())) { + if rx_unlikely (!status.ok()) { throw status; } if (rx_likely(db != nullptr)) { @@ -700,7 +701,7 @@ RPCQrWatcher::Ref RPCServer::createQueryResults(cproto::Context &ctx, RPCQrId &i } } - if (rx_unlikely(data->results.size() >= cproto::kMaxConcurentQueries)) { + if rx_unlikely (data->results.size() >= cproto::kMaxConcurentQueries) { for (unsigned idx = 0; idx < data->results.size(); ++idx) { RPCQrId tmpQrId{data->results[idx].main, data->results[idx].uid}; assertrx(tmpQrId.main >= 0); diff --git a/cpp_src/server/rpcupdatespusher.cc b/cpp_src/server/rpcupdatespusher.cc index c842aa240..a7f7e279e 100644 --- a/cpp_src/server/rpcupdatespusher.cc +++ b/cpp_src/server/rpcupdatespusher.cc @@ -21,10 +21,11 @@ void RPCUpdatesPusher::OnWALUpdate(LSNPair LSNs, std::string_view nsName, const pwalRec = walRec.GetShared(int64_t(LSNs.upstreamLSN_), int64_t(LSNs.originLSN_), nsName); } - writer_->CallRPC({[](IRPCCall *self, CmdCode &cmd, Args &args) { + writer_->CallRPC({[](IRPCCall *self, CmdCode &cmd, std::string_view &ns, Args &args) { auto unpacked = SharedWALRecord(self->data_).Unpack(); cmd = kCmdUpdates; args = {Arg(unpacked.upstreamLSN), Arg(unpacked.nsName), Arg(unpacked.pwalRec), Arg(unpacked.originLSN)}; + ns = std::string_view(args[1]); }, pwalRec.packed_ diff --git a/cpp_src/server/server.cc b/cpp_src/server/server.cc index f753676e4..5b42a8614 100644 --- a/cpp_src/server/server.cc +++ b/cpp_src/server/server.cc @@ -10,10 +10,10 @@ Error Server::InitFromFile(const char *filePath) { return impl_->InitFromFile(fi Error Server::InitFromYAML(const std::string &yaml) { return impl_->InitFromYAML(yaml); } int Server::Start() { return impl_->Start(); } void Server::Stop() { return impl_->Stop(); } -void Server::EnableHandleSignals(bool enable) { impl_->EnableHandleSignals(enable); } -DBManager &Server::GetDBManager() { return impl_->GetDBManager(); } -bool Server::IsReady() { return impl_->IsReady(); } -bool Server::IsRunning() { return impl_->IsRunning(); } +void Server::EnableHandleSignals(bool enable) noexcept { impl_->EnableHandleSignals(enable); } +DBManager &Server::GetDBManager() noexcept { return impl_->GetDBManager(); } +bool Server::IsReady() const noexcept { return impl_->IsReady(); } +bool Server::IsRunning() const noexcept { return impl_->IsRunning(); } void Server::ReopenLogFiles() { impl_->ReopenLogFiles(); } } // namespace reindexer_server diff --git a/cpp_src/server/server.h b/cpp_src/server/server.h index 10e316b42..e9034e425 100644 --- a/cpp_src/server/server.h +++ b/cpp_src/server/server.h @@ -22,10 +22,10 @@ class Server { Error InitFromFile(const char* filepath); int Start(); void Stop(); - void EnableHandleSignals(bool enable = true); - DBManager& GetDBManager(); - bool IsReady(); - bool IsRunning(); + void EnableHandleSignals(bool enable = true) noexcept; + DBManager& GetDBManager() noexcept; + bool IsReady() const noexcept; + bool IsRunning() const noexcept; void ReopenLogFiles(); protected: diff --git a/cpp_src/server/serverimpl.cc b/cpp_src/server/serverimpl.cc index f1e4c5028..cc7ed0255 100644 --- a/cpp_src/server/serverimpl.cc +++ b/cpp_src/server/serverimpl.cc @@ -29,9 +29,9 @@ #endif #ifdef LINK_RESOURCES #include -void init_resources() { CMRC_INIT(reindexer_server_resources); } +static void init_resources() { CMRC_INIT(reindexer_server_resources); } #else -void init_resources() {} +static void init_resources() {} #endif #if defined(WITH_GRPC) @@ -47,7 +47,7 @@ namespace reindexer_server { using reindexer::fs::GetDirPath; using reindexer::logLevelFromString; -ServerImpl::ServerImpl([[maybe_unused]] ServerMode mode) +ServerImpl::ServerImpl(ServerMode mode) : #ifdef REINDEX_WITH_GPERFTOOLS config_(alloc_ext::TCMallocIsAvailable()), @@ -56,12 +56,8 @@ ServerImpl::ServerImpl([[maybe_unused]] ServerMode mode) #endif coreLogLevel_(LogNone), storageLoaded_(false), - running_(false) -#ifndef REINDEX_WITH_ASAN - , - mode_(mode) -#endif // REINDEX_WITH_ASAN -{ + running_(false), + mode_(mode) { async_.set(loop_); } @@ -516,7 +512,9 @@ void ServerImpl::initCoreLogger() { } } }; - if (coreLogLevel_ && logger.lock()) reindexer::logInstallWriter(callback); + if (coreLogLevel_ && logger.lock()) { + reindexer::logInstallWriter(callback, mode_ == ServerMode::Standalone ? LoggerPolicy::WithoutLocks : LoggerPolicy::WithLocks); + } } ServerImpl::~ServerImpl() { @@ -525,7 +523,9 @@ ServerImpl::~ServerImpl() { rxAllowNamespaceLeak = true; } #endif - if (coreLogLevel_) reindexer::logInstallWriter(nullptr); + if (coreLogLevel_) { + logInstallWriter(nullptr, mode_ == ServerMode::Standalone ? LoggerPolicy::WithoutLocks : LoggerPolicy::WithLocks); + } async_.reset(); } diff --git a/cpp_src/server/serverimpl.h b/cpp_src/server/serverimpl.h index 186559b5a..bf0444d61 100644 --- a/cpp_src/server/serverimpl.h +++ b/cpp_src/server/serverimpl.h @@ -35,10 +35,10 @@ class ServerImpl { Error InitFromFile(const char* filepath); int Start(); void Stop(); - void EnableHandleSignals(bool enable = true) { enableHandleSignals_ = enable; } - DBManager& GetDBManager() { return *dbMgr_; } - bool IsReady() const { return storageLoaded_.load(); } - bool IsRunning() const { return running_.load(); } + void EnableHandleSignals(bool enable = true) noexcept { enableHandleSignals_ = enable; } + DBManager& GetDBManager() noexcept { return *dbMgr_; } + bool IsReady() const noexcept { return storageLoaded_.load(); } + bool IsRunning() const noexcept { return running_.load(); } void ReopenLogFiles(); protected: @@ -68,8 +68,6 @@ class ServerImpl { bool enableHandleSignals_ = false; ev::async async_; ev::dynamic_loop loop_; -#ifndef REINDEX_WITH_ASAN ServerMode mode_ = ServerMode::Builtin; -#endif // REINDEX_WITH_ASAN }; } // namespace reindexer_server diff --git a/cpp_src/tools/customhash.cc b/cpp_src/tools/customhash.cc index e71727ba9..00dc80f28 100644 --- a/cpp_src/tools/customhash.cc +++ b/cpp_src/tools/customhash.cc @@ -124,7 +124,7 @@ static uint32_t _Hash_bytes_collate_utf8(const void* ptr, uint32_t len) noexcept return hash; } -uint32_t Hash(const wstring& s) noexcept { return _Hash_bytes(s.data(), s.length() * sizeof(wchar_t)); } +uint32_t Hash(const std::wstring& s) noexcept { return _Hash_bytes(s.data(), s.length() * sizeof(wchar_t)); } template <> uint32_t collateHash(std::string_view s) noexcept { return _Hash_bytes_collate_ascii(s.data(), s.length()); diff --git a/cpp_src/tools/customlocal.cc b/cpp_src/tools/customlocal.cc index 499a82781..a5cfbcf75 100644 --- a/cpp_src/tools/customlocal.cc +++ b/cpp_src/tools/customlocal.cc @@ -1,152 +1,261 @@ #include "customlocal.h" -#include #include -#include +#include namespace reindexer { +// {lower, upper} +constexpr std::pair kAlphabet[] = { + // clang-format off + // Basic Latin + {0x0061, 0x0041}, {0x0062, 0x0042}, {0x0063, 0x0043}, {0x0064, 0x0044}, {0x0065, 0x0045}, {0x0066, 0x0046}, {0x0067, 0x0047}, + {0x0068, 0x0048}, {0x0069, 0x0049}, {0x006A, 0x004A}, {0x006B, 0x004B}, {0x006C, 0x004C}, {0x006D, 0x004D}, {0x006E, 0x004E}, + {0x006F, 0x004F}, {0x0070, 0x0050}, {0x0071, 0x0051}, {0x0072, 0x0052}, {0x0073, 0x0053}, {0x0074, 0x0054}, {0x0075, 0x0055}, + {0x0076, 0x0056}, {0x0077, 0x0057}, {0x0078, 0x0058}, {0x0079, 0x0059}, {0x007A, 0x005A}, + //Latin-1 Supplement + {0x00E0, 0x00C0}, {0x00E1, 0x00C1}, {0x00E2, 0x00C2}, {0x00E3, 0x00C3}, {0x00E4, 0x00C4}, {0x00E5, 0x00C5}, {0x00E6, 0x00C6}, + {0x00E7, 0x00C7}, {0x00E8, 0x00C8}, {0x00E9, 0x00C9}, {0x00EA, 0x00CA}, {0x00EB, 0x00CB}, {0x00EC, 0x00CC}, {0x00ED, 0x00CD}, + {0x00EE, 0x00CE}, {0x00EF, 0x00CF}, {0x00F0, 0x00D0}, {0x00F1, 0x00D1}, {0x00F2, 0x00D2}, {0x00F3, 0x00D3}, {0x00F4, 0x00D4}, + {0x00F5, 0x00D5}, {0x00F6, 0x00D6}, {0x00F8, 0x00D8}, {0x00F9, 0x00D9}, {0x00FA, 0x00DA}, {0x00FB, 0x00DB}, {0x00FC, 0x00DC}, + {0x00FD, 0x00DD}, {0x00FE, 0x00DE}, {0x00FF, 0x0178}, + // Latin Extended-A + {0x0101, 0x0100}, {0x0103, 0x0102}, {0x0105, 0x0104}, {0x0107, 0x0106}, {0x0109, 0x0108}, {0x010B, 0x010A}, {0x010D, 0x010C}, + {0x010F, 0x010E}, {0x0111, 0x0110}, {0x0113, 0x0112}, {0x0115, 0x0114}, {0x0117, 0x0116}, {0x0119, 0x0118}, {0x011B, 0x011A}, + {0x011D, 0x011C}, {0x011F, 0x011E}, {0x0121, 0x0120}, {0x0123, 0x0122}, {0x0125, 0x0124}, {0x0127, 0x0126}, {0x0129, 0x0128}, + {0x012B, 0x012A}, {0x012D, 0x012C}, {0x012F, 0x012E}, {0x0131, 0x0049}, {0x0133, 0x0132}, {0x0135, 0x0134}, {0x0137, 0x0136}, + {0x013A, 0x0139}, {0x013C, 0x013B}, {0x013E, 0x013D}, {0x0140, 0x013F}, {0x0142, 0x0141}, {0x0144, 0x0143}, {0x0146, 0x0145}, + {0x0148, 0x0147}, {0x014B, 0x014A}, {0x014D, 0x014C}, {0x014F, 0x014E}, {0x0151, 0x0150}, {0x0153, 0x0152}, {0x0155, 0x0154}, + {0x0157, 0x0156}, {0x0159, 0x0158}, {0x015B, 0x015A}, {0x015D, 0x015C}, {0x015F, 0x015E}, {0x0161, 0x0160}, {0x0163, 0x0162}, + {0x0165, 0x0164}, {0x0167, 0x0166}, {0x0169, 0x0168}, {0x016B, 0x016A}, {0x016D, 0x016C}, {0x016F, 0x016E}, {0x0171, 0x0170}, + {0x0173, 0x0172}, {0x0175, 0x0174}, {0x0177, 0x0176}, {0x017A, 0x0179}, {0x017C, 0x017B}, {0x017E, 0x017D}, + //Latin Extended-B + {0x0183, 0x0182}, {0x0185, 0x0184}, {0x0188, 0x0187}, {0x018C, 0x018B}, {0x0192, 0x0191}, {0x0199, 0x0198}, {0x01A1, 0x01A0}, + {0x01A3, 0x01A2}, {0x01A5, 0x01A4}, {0x01A8, 0x01A7}, {0x01AD, 0x01AC}, {0x01B0, 0x01AF}, {0x01B4, 0x01B3}, {0x01B6, 0x01B5}, + {0x01B9, 0x01B8}, {0x01BD, 0x01BC}, {0x01C6, 0x01C4}, {0x01C9, 0x01C7}, {0x01CC, 0x01CA}, {0x01CE, 0x01CD}, {0x01D0, 0x01CF}, + {0x01D2, 0x01D1}, {0x01D4, 0x01D3}, {0x01D6, 0x01D5}, {0x01D8, 0x01D7}, {0x01DA, 0x01D9}, {0x01DC, 0x01DB}, {0x01DF, 0x01DE}, + {0x01E1, 0x01E0}, {0x01E3, 0x01E2}, {0x01E5, 0x01E4}, {0x01E7, 0x01E6}, {0x01E9, 0x01E8}, {0x01EB, 0x01EA}, {0x01ED, 0x01EC}, + {0x01EF, 0x01EE}, {0x01F3, 0x01F1}, {0x01F5, 0x01F4}, {0x01FB, 0x01FA}, {0x01FD, 0x01FC}, {0x01FF, 0x01FE}, {0x0201, 0x0200}, + {0x0203, 0x0202}, {0x0205, 0x0204}, {0x0207, 0x0206}, {0x0209, 0x0208}, {0x020B, 0x020A}, {0x020D, 0x020C}, {0x020F, 0x020E}, + {0x0211, 0x0210}, {0x0213, 0x0212}, {0x0215, 0x0214}, {0x0217, 0x0216}, + // IPA Extensions + {0x0253, 0x0181}, {0x0254, 0x0186}, {0x0257, 0x018A}, {0x0258, 0x018E}, {0x0259, 0x018F}, {0x025B, 0x0190}, {0x0260, 0x0193}, + {0x0263, 0x0194}, {0x0268, 0x0197}, {0x0269, 0x0196}, {0x026F, 0x019C}, {0x0272, 0x019D}, {0x0275, 0x019F}, {0x0283, 0x01A9}, + {0x0288, 0x01AE}, {0x028A, 0x01B1}, {0x028B, 0x01B2}, {0x0292, 0x01B7}, + // Greek and Coptic + {0x03AC, 0x0386}, {0x03AD, 0x0388}, {0x03AE, 0x0389}, {0x03AF, 0x038A}, {0x03B1, 0x0391}, {0x03B2, 0x0392}, {0x03B3, 0x0393}, + {0x03B4, 0x0394}, {0x03B5, 0x0395}, {0x03B6, 0x0396}, {0x03B7, 0x0397}, {0x03B8, 0x0398}, {0x03B9, 0x0399}, {0x03BA, 0x039A}, + {0x03BB, 0x039B}, {0x03BC, 0x039C}, {0x03BD, 0x039D}, {0x03BE, 0x039E}, {0x03BF, 0x039F}, {0x03C0, 0x03A0}, {0x03C1, 0x03A1}, + {0x03C3, 0x03A3}, {0x03C4, 0x03A4}, {0x03C5, 0x03A5}, {0x03C6, 0x03A6}, {0x03C7, 0x03A7}, {0x03C8, 0x03A8}, {0x03C9, 0x03A9}, + {0x03CA, 0x03AA}, {0x03CB, 0x03AB}, {0x03CC, 0x038C}, {0x03CD, 0x038E}, {0x03CE, 0x038F}, {0x03E3, 0x03E2}, {0x03E5, 0x03E4}, + {0x03E7, 0x03E6}, {0x03E9, 0x03E8}, {0x03EB, 0x03EA}, {0x03ED, 0x03EC}, {0x03EF, 0x03EE}, + // Cyrillic + {0x0430, 0x0410}, {0x0431, 0x0411}, {0x0432, 0x0412}, {0x0433, 0x0413}, {0x0434, 0x0414}, {0x0435, 0x0415}, {0x0436, 0x0416}, + {0x0437, 0x0417}, {0x0438, 0x0418}, {0x0439, 0x0419}, {0x043A, 0x041A}, {0x043B, 0x041B}, {0x043C, 0x041C}, {0x043D, 0x041D}, + {0x043E, 0x041E}, {0x043F, 0x041F}, {0x0440, 0x0420}, {0x0441, 0x0421}, {0x0442, 0x0422}, {0x0443, 0x0423}, {0x0444, 0x0424}, + {0x0445, 0x0425}, {0x0446, 0x0426}, {0x0447, 0x0427}, {0x0448, 0x0428}, {0x0449, 0x0429}, {0x044A, 0x042A}, {0x044B, 0x042B}, + {0x044C, 0x042C}, {0x044D, 0x042D}, {0x044E, 0x042E}, {0x044F, 0x042F}, {0x0451, 0x0401}, {0x0452, 0x0402}, {0x0453, 0x0403}, + {0x0454, 0x0404}, {0x0455, 0x0405}, {0x0456, 0x0406}, {0x0457, 0x0407}, {0x0458, 0x0408}, {0x0459, 0x0409}, {0x045A, 0x040A}, + {0x045B, 0x040B}, {0x045C, 0x040C}, {0x045E, 0x040E}, {0x045F, 0x040F}, {0x0461, 0x0460}, {0x0463, 0x0462}, {0x0465, 0x0464}, + {0x0467, 0x0466}, {0x0469, 0x0468}, {0x046B, 0x046A}, {0x046D, 0x046C}, {0x046F, 0x046E}, {0x0471, 0x0470}, {0x0473, 0x0472}, + {0x0475, 0x0474}, {0x0477, 0x0476}, {0x0479, 0x0478}, {0x047B, 0x047A}, {0x047D, 0x047C}, {0x047F, 0x047E}, {0x0481, 0x0480}, + {0x0491, 0x0490}, {0x0493, 0x0492}, {0x0495, 0x0494}, {0x0497, 0x0496}, {0x0499, 0x0498}, {0x049B, 0x049A}, {0x049D, 0x049C}, + {0x049F, 0x049E}, {0x04A1, 0x04A0}, {0x04A3, 0x04A2}, {0x04A5, 0x04A4}, {0x04A7, 0x04A6}, {0x04A9, 0x04A8}, {0x04AB, 0x04AA}, + {0x04AD, 0x04AC}, {0x04AF, 0x04AE}, {0x04B1, 0x04B0}, {0x04B3, 0x04B2}, {0x04B5, 0x04B4}, {0x04B7, 0x04B6}, {0x04B9, 0x04B8}, + {0x04BB, 0x04BA}, {0x04BD, 0x04BC}, {0x04BF, 0x04BE}, {0x04C2, 0x04C1}, {0x04C4, 0x04C3}, {0x04C8, 0x04C7}, {0x04CC, 0x04CB}, + {0x04D1, 0x04D0}, {0x04D3, 0x04D2}, {0x04D5, 0x04D4}, {0x04D7, 0x04D6}, {0x04D9, 0x04D8}, {0x04DB, 0x04DA}, {0x04DD, 0x04DC}, + {0x04DF, 0x04DE}, {0x04E1, 0x04E0}, {0x04E3, 0x04E2}, {0x04E5, 0x04E4}, {0x04E7, 0x04E6}, {0x04E9, 0x04E8}, {0x04EB, 0x04EA}, + {0x04EF, 0x04EE}, {0x04F1, 0x04F0}, {0x04F3, 0x04F2}, {0x04F5, 0x04F4}, {0x04F9, 0x04F8}, + // Armenian + {0x0561, 0x0531}, {0x0562, 0x0532}, {0x0563, 0x0533}, {0x0564, 0x0534}, {0x0565, 0x0535}, {0x0566, 0x0536}, {0x0567, 0x0537}, + {0x0568, 0x0538}, {0x0569, 0x0539}, {0x056A, 0x053A}, {0x056B, 0x053B}, {0x056C, 0x053C}, {0x056D, 0x053D}, {0x056E, 0x053E}, + {0x056F, 0x053F}, {0x0570, 0x0540}, {0x0571, 0x0541}, {0x0572, 0x0542}, {0x0573, 0x0543}, {0x0574, 0x0544}, {0x0575, 0x0545}, + {0x0576, 0x0546}, {0x0577, 0x0547}, {0x0578, 0x0548}, {0x0579, 0x0549}, {0x057A, 0x054A}, {0x057B, 0x054B}, {0x057C, 0x054C}, + {0x057D, 0x054D}, {0x057E, 0x054E}, {0x057F, 0x054F}, {0x0580, 0x0550}, {0x0581, 0x0551}, {0x0582, 0x0552}, {0x0583, 0x0553}, + {0x0584, 0x0554}, {0x0585, 0x0555}, {0x0586, 0x0556}, + // Hebrew + {0x0591, 0x0591}, {0x0592, 0x0592}, {0x0593, 0x0593}, {0x0594, 0x0594}, {0x0595, 0x0595}, {0x0596, 0x0596}, {0x0597, 0x0597}, + {0x0598, 0x0598}, {0x0599, 0x0599}, {0x059A, 0x059A}, {0x059B, 0x059B}, {0x059C, 0x059C}, {0x059D, 0x059D}, {0x059E, 0x059E}, + {0x059F, 0x059F}, {0x05A0, 0x05A0}, {0x05A1, 0x05A1}, {0x05A2, 0x05A2}, {0x05A3, 0x05A3}, {0x05A4, 0x05A4}, {0x05A5, 0x05A5}, + {0x05A6, 0x05A6}, {0x05A7, 0x05A7}, {0x05A8, 0x05A8}, {0x05A9, 0x05A9}, {0x05AA, 0x05AA}, {0x05AB, 0x05AB}, {0x05AC, 0x05AC}, + {0x05AD, 0x05AD}, {0x05AE, 0x05AE}, {0x05AF, 0x05AF}, {0x05B0, 0x05B0}, {0x05B1, 0x05B1}, {0x05B2, 0x05B2}, {0x05B3, 0x05B3}, + {0x05B4, 0x05B4}, {0x05B5, 0x05B5}, {0x05B6, 0x05B6}, {0x05B7, 0x05B7}, {0x05B8, 0x05B8}, {0x05B9, 0x05B9}, {0x05BA, 0x05BA}, + {0x05BB, 0x05BB}, {0x05BC, 0x05BC}, {0x05BD, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C1}, {0x05C2, 0x05C2}, {0x05C4, 0x05C4}, + {0x05C5, 0x05C5}, {0x05C7, 0x05C7}, {0x05D0, 0x05D0}, {0x05D1, 0x05D1}, {0x05D2, 0x05D2}, {0x05D3, 0x05D3}, {0x05D4, 0x05D4}, + {0x05D5, 0x05D5}, {0x05D6, 0x05D6}, {0x05D7, 0x05D7}, {0x05D8, 0x05D8}, {0x05D9, 0x05D9}, {0x05DA, 0x05DA}, {0x05DB, 0x05DB}, + {0x05DC, 0x05DC}, {0x05DD, 0x05DD}, {0x05DE, 0x05DE}, {0x05DF, 0x05DF}, {0x05E0, 0x05E0}, {0x05E1, 0x05E1}, {0x05E2, 0x05E2}, + {0x05E3, 0x05E3}, {0x05E4, 0x05E4}, {0x05E5, 0x05E5}, {0x05E6, 0x05E6}, {0x05E7, 0x05E7}, {0x05E8, 0x05E8}, {0x05E9, 0x05E9}, + {0x05EA, 0x05EA}, {0x05F0, 0x05F0}, {0x05F1, 0x05F1}, {0x05F2, 0x05F2}, + // Arabic + {0x0610, 0x0610}, {0x0611, 0x0611}, {0x0612, 0x0612}, {0x0613, 0x0613}, {0x0614, 0x0614}, {0x0615, 0x0615}, {0x0616, 0x0616}, + {0x0617, 0x0617}, {0x0618, 0x0618}, {0x0619, 0x0619}, {0x061A, 0x061A}, {0x0620, 0x0620}, {0x0621, 0x0621}, {0x0622, 0x0622}, + {0x0623, 0x0623}, {0x0624, 0x0624}, {0x0625, 0x0625}, {0x0626, 0x0626}, {0x0627, 0x0627}, {0x0628, 0x0628}, {0x0629, 0x0629}, + {0x062A, 0x062A}, {0x062B, 0x062B}, {0x062C, 0x062C}, {0x062D, 0x062D}, {0x062E, 0x062E}, {0x062F, 0x062F}, {0x0630, 0x0630}, + {0x0631, 0x0631}, {0x0632, 0x0632}, {0x0633, 0x0633}, {0x0634, 0x0634}, {0x0635, 0x0635}, {0x0636, 0x0636}, {0x0637, 0x0637}, + {0x0638, 0x0638}, {0x0639, 0x0639}, {0x063A, 0x063A}, {0x063B, 0x063B}, {0x063C, 0x063C}, {0x063D, 0x063D}, {0x063E, 0x063E}, + {0x063F, 0x063F}, {0x0640, 0x0640}, {0x0641, 0x0641}, {0x0642, 0x0642}, {0x0643, 0x0643}, {0x0644, 0x0644}, {0x0645, 0x0645}, + {0x0646, 0x0646}, {0x0647, 0x0647}, {0x0648, 0x0648}, {0x0649, 0x0649}, {0x064A, 0x064A}, {0x064B, 0x064B}, {0x064C, 0x064C}, + {0x064D, 0x064D}, {0x064E, 0x064E}, {0x064F, 0x064F}, {0x0650, 0x0650}, {0x0651, 0x0651}, {0x0652, 0x0652}, {0x0653, 0x0653}, + {0x0654, 0x0654}, {0x0655, 0x0655}, {0x0656, 0x0656}, {0x0657, 0x0657}, {0x0658, 0x0658}, {0x0659, 0x0659}, {0x065A, 0x065A}, + {0x065B, 0x065B}, {0x065C, 0x065C}, {0x065D, 0x065D}, {0x065E, 0x065E}, {0x065F, 0x065F}, {0x066E, 0x066E}, {0x066F, 0x066F}, + {0x0670, 0x0670}, {0x0671, 0x0671}, {0x0672, 0x0672}, {0x0673, 0x0673}, {0x0674, 0x0674}, {0x0675, 0x0675}, {0x0676, 0x0676}, + {0x0677, 0x0677}, {0x0678, 0x0678}, {0x0679, 0x0679}, {0x067A, 0x067A}, {0x067B, 0x067B}, {0x067C, 0x067C}, {0x067D, 0x067D}, + {0x067E, 0x067E}, {0x067F, 0x067F}, {0x0680, 0x0680}, {0x0681, 0x0681}, {0x0682, 0x0682}, {0x0683, 0x0683}, {0x0684, 0x0684}, + {0x0685, 0x0685}, {0x0686, 0x0686}, {0x0687, 0x0687}, {0x0688, 0x0688}, {0x0689, 0x0689}, {0x068A, 0x068A}, {0x068B, 0x068B}, + {0x068C, 0x068C}, {0x068D, 0x068D}, {0x068E, 0x068E}, {0x068F, 0x068F}, {0x0690, 0x0690}, {0x0691, 0x0691}, {0x0692, 0x0692}, + {0x0693, 0x0693}, {0x0694, 0x0694}, {0x0695, 0x0695}, {0x0696, 0x0696}, {0x0697, 0x0697}, {0x0698, 0x0698}, {0x0699, 0x0699}, + {0x069A, 0x069A}, {0x069B, 0x069B}, {0x069C, 0x069C}, {0x069D, 0x069D}, {0x069E, 0x069E}, {0x069F, 0x069F}, {0x06A0, 0x06A0}, + {0x06A1, 0x06A1}, {0x06A2, 0x06A2}, {0x06A3, 0x06A3}, {0x06A4, 0x06A4}, {0x06A5, 0x06A5}, {0x06A6, 0x06A6}, {0x06A7, 0x06A7}, + {0x06A8, 0x06A8}, {0x06A9, 0x06A9}, {0x06AA, 0x06AA}, {0x06AB, 0x06AB}, {0x06AC, 0x06AC}, {0x06AD, 0x06AD}, {0x06AE, 0x06AE}, + {0x06AF, 0x06AF}, {0x06B0, 0x06B0}, {0x06B1, 0x06B1}, {0x06B2, 0x06B2}, {0x06B3, 0x06B3}, {0x06B4, 0x06B4}, {0x06B5, 0x06B5}, + {0x06B6, 0x06B6}, {0x06B7, 0x06B7}, {0x06B8, 0x06B8}, {0x06B9, 0x06B9}, {0x06BA, 0x06BA}, {0x06BB, 0x06BB}, {0x06BC, 0x06BC}, + {0x06BD, 0x06BD}, {0x06BE, 0x06BE}, {0x06BF, 0x06BF}, {0x06C0, 0x06C0}, {0x06C1, 0x06C1}, {0x06C2, 0x06C2}, {0x06C3, 0x06C3}, + {0x06C4, 0x06C4}, {0x06C5, 0x06C5}, {0x06C6, 0x06C6}, {0x06C7, 0x06C7}, {0x06C8, 0x06C8}, {0x06C9, 0x06C9}, {0x06CA, 0x06CA}, + {0x06CB, 0x06CB}, {0x06CC, 0x06CC}, {0x06CD, 0x06CD}, {0x06CE, 0x06CE}, {0x06CF, 0x06CF}, {0x06D0, 0x06D0}, {0x06D1, 0x06D1}, + {0x06D2, 0x06D2}, {0x06D3, 0x06D3}, {0x06D5, 0x06D5}, {0x06D6, 0x06D6}, {0x06D7, 0x06D7}, {0x06D8, 0x06D8}, {0x06D9, 0x06D9}, + {0x06DA, 0x06DA}, {0x06DB, 0x06DB}, {0x06DC, 0x06DC}, {0x06DF, 0x06DF}, {0x06E0, 0x06E0}, {0x06E1, 0x06E1}, {0x06E2, 0x06E2}, + {0x06E3, 0x06E3}, {0x06E4, 0x06E4}, {0x06E5, 0x06E5}, {0x06E6, 0x06E6}, {0x06E7, 0x06E7}, {0x06E8, 0x06E8}, {0x06EA, 0x06EA}, + {0x06EB, 0x06EB}, {0x06EC, 0x06EC}, {0x06ED, 0x06ED}, {0x06EE, 0x06EE}, {0x06EF, 0x06EF}, {0x06FA, 0x06FA}, {0x06FB, 0x06FB}, + {0x06FC, 0x06FC}, {0x06FF, 0x06FF}, + // Devanagari + {0x0900, 0x0900}, {0x0901, 0x0901}, {0x0902, 0x0902}, {0x0903, 0x0903}, {0x0904, 0x0904}, {0x0905, 0x0905}, {0x0906, 0x0906}, + {0x0907, 0x0907}, {0x0908, 0x0908}, {0x0909, 0x0909}, {0x090A, 0x090A}, {0x090B, 0x090B}, {0x090C, 0x090C}, {0x090D, 0x090D}, + {0x090E, 0x090E}, {0x090F, 0x090F}, {0x0910, 0x0910}, {0x0911, 0x0911}, {0x0912, 0x0912}, {0x0913, 0x0913}, {0x0914, 0x0914}, + {0x0915, 0x0915}, {0x0916, 0x0916}, {0x0917, 0x0917}, {0x0918, 0x0918}, {0x0919, 0x0919}, {0x091A, 0x091A}, {0x091B, 0x091B}, + {0x091C, 0x091C}, {0x091D, 0x091D}, {0x091E, 0x091E}, {0x091F, 0x091F}, {0x0920, 0x0920}, {0x0921, 0x0921}, {0x0922, 0x0922}, + {0x0923, 0x0923}, {0x0924, 0x0924}, {0x0925, 0x0925}, {0x0926, 0x0926}, {0x0927, 0x0927}, {0x0928, 0x0928}, {0x0929, 0x0929}, + {0x092A, 0x092A}, {0x092B, 0x092B}, {0x092C, 0x092C}, {0x092D, 0x092D}, {0x092E, 0x092E}, {0x092F, 0x092F}, {0x0930, 0x0930}, + {0x0931, 0x0931}, {0x0932, 0x0932}, {0x0933, 0x0933}, {0x0934, 0x0934}, {0x0935, 0x0935}, {0x0936, 0x0936}, {0x0937, 0x0937}, + {0x0938, 0x0938}, {0x0939, 0x0939}, {0x093A, 0x093A}, {0x093B, 0x093B}, {0x093C, 0x093C}, {0x093D, 0x093D}, {0x093E, 0x093E}, + {0x093F, 0x093F}, {0x0940, 0x0940}, {0x0941, 0x0941}, {0x0942, 0x0942}, {0x0943, 0x0943}, {0x0944, 0x0944}, {0x0945, 0x0945}, + {0x0946, 0x0946}, {0x0947, 0x0947}, {0x0948, 0x0948}, {0x0949, 0x0949}, {0x094A, 0x094A}, {0x094B, 0x094B}, {0x094C, 0x094C}, + {0x094D, 0x094D}, {0x094E, 0x094E}, {0x094F, 0x094F}, {0x0950, 0x0950}, {0x0951, 0x0951}, {0x0952, 0x0952}, {0x0953, 0x0953}, + {0x0954, 0x0954}, {0x0955, 0x0955}, {0x0956, 0x0956}, {0x0957, 0x0957}, {0x0958, 0x0958}, {0x0959, 0x0959}, {0x095A, 0x095A}, + {0x095B, 0x095B}, {0x095C, 0x095C}, {0x095D, 0x095D}, {0x095E, 0x095E}, {0x095F, 0x095F}, {0x0960, 0x0960}, {0x0961, 0x0961}, + {0x0962, 0x0962}, {0x0963, 0x0963}, {0x0971, 0x0971}, {0x0972, 0x0972}, {0x0973, 0x0973}, {0x0974, 0x0974}, {0x0975, 0x0975}, + {0x0976, 0x0976}, {0x0977, 0x0977}, {0x0978, 0x0978}, {0x0979, 0x0979}, {0x097A, 0x097A}, {0x097B, 0x097B}, {0x097C, 0x097C}, + {0x097D, 0x097D}, {0x097E, 0x097E}, {0x097F, 0x097F}, + // Georgian + {0x10D0, 0x10A0}, {0x10D1, 0x10A1}, {0x10D2, 0x10A2}, {0x10D3, 0x10A3}, {0x10D4, 0x10A4}, {0x10D5, 0x10A5}, {0x10D6, 0x10A6}, + {0x10D7, 0x10A7}, {0x10D8, 0x10A8}, {0x10D9, 0x10A9}, {0x10DA, 0x10AA}, {0x10DB, 0x10AB}, {0x10DC, 0x10AC}, {0x10DD, 0x10AD}, + {0x10DE, 0x10AE}, {0x10DF, 0x10AF}, {0x10E0, 0x10B0}, {0x10E1, 0x10B1}, {0x10E2, 0x10B2}, {0x10E3, 0x10B3}, {0x10E4, 0x10B4}, + {0x10E5, 0x10B5}, {0x10E6, 0x10B6}, {0x10E7, 0x10B7}, {0x10E8, 0x10B8}, {0x10E9, 0x10B9}, {0x10EA, 0x10BA}, {0x10EB, 0x10BB}, + {0x10EC, 0x10BC}, {0x10ED, 0x10BD}, {0x10EE, 0x10BE}, {0x10EF, 0x10BF}, {0x10F0, 0x10C0}, {0x10F1, 0x10C1}, {0x10F2, 0x10C2}, + {0x10F3, 0x10C3}, {0x10F4, 0x10C4}, {0x10F5, 0x10C5}, + // Latin Extended Additional + {0x1E01, 0x1E00}, {0x1E03, 0x1E02}, {0x1E05, 0x1E04}, {0x1E07, 0x1E06}, {0x1E09, 0x1E08}, {0x1E0B, 0x1E0A}, {0x1E0D, 0x1E0C}, + {0x1E0F, 0x1E0E}, {0x1E11, 0x1E10}, {0x1E13, 0x1E12}, {0x1E15, 0x1E14}, {0x1E17, 0x1E16}, {0x1E19, 0x1E18}, {0x1E1B, 0x1E1A}, + {0x1E1D, 0x1E1C}, {0x1E1F, 0x1E1E}, {0x1E21, 0x1E20}, {0x1E23, 0x1E22}, {0x1E25, 0x1E24}, {0x1E27, 0x1E26}, {0x1E29, 0x1E28}, + {0x1E2B, 0x1E2A}, {0x1E2D, 0x1E2C}, {0x1E2F, 0x1E2E}, {0x1E31, 0x1E30}, {0x1E33, 0x1E32}, {0x1E35, 0x1E34}, {0x1E37, 0x1E36}, + {0x1E39, 0x1E38}, {0x1E3B, 0x1E3A}, {0x1E3D, 0x1E3C}, {0x1E3F, 0x1E3E}, {0x1E41, 0x1E40}, {0x1E43, 0x1E42}, {0x1E45, 0x1E44}, + {0x1E47, 0x1E46}, {0x1E49, 0x1E48}, {0x1E4B, 0x1E4A}, {0x1E4D, 0x1E4C}, {0x1E4F, 0x1E4E}, {0x1E51, 0x1E50}, {0x1E53, 0x1E52}, + {0x1E55, 0x1E54}, {0x1E57, 0x1E56}, {0x1E59, 0x1E58}, {0x1E5B, 0x1E5A}, {0x1E5D, 0x1E5C}, {0x1E5F, 0x1E5E}, {0x1E61, 0x1E60}, + {0x1E63, 0x1E62}, {0x1E65, 0x1E64}, {0x1E67, 0x1E66}, {0x1E69, 0x1E68}, {0x1E6B, 0x1E6A}, {0x1E6D, 0x1E6C}, {0x1E6F, 0x1E6E}, + {0x1E71, 0x1E70}, {0x1E73, 0x1E72}, {0x1E75, 0x1E74}, {0x1E77, 0x1E76}, {0x1E79, 0x1E78}, {0x1E7B, 0x1E7A}, {0x1E7D, 0x1E7C}, + {0x1E7F, 0x1E7E}, {0x1E81, 0x1E80}, {0x1E83, 0x1E82}, {0x1E85, 0x1E84}, {0x1E87, 0x1E86}, {0x1E89, 0x1E88}, {0x1E8B, 0x1E8A}, + {0x1E8D, 0x1E8C}, {0x1E8F, 0x1E8E}, {0x1E91, 0x1E90}, {0x1E93, 0x1E92}, {0x1E95, 0x1E94}, {0x1EA1, 0x1EA0}, {0x1EA3, 0x1EA2}, + {0x1EA5, 0x1EA4}, {0x1EA7, 0x1EA6}, {0x1EA9, 0x1EA8}, {0x1EAB, 0x1EAA}, {0x1EAD, 0x1EAC}, {0x1EAF, 0x1EAE}, {0x1EB1, 0x1EB0}, + {0x1EB3, 0x1EB2}, {0x1EB5, 0x1EB4}, {0x1EB7, 0x1EB6}, {0x1EB9, 0x1EB8}, {0x1EBB, 0x1EBA}, {0x1EBD, 0x1EBC}, {0x1EBF, 0x1EBE}, + {0x1EC1, 0x1EC0}, {0x1EC3, 0x1EC2}, {0x1EC5, 0x1EC4}, {0x1EC7, 0x1EC6}, {0x1EC9, 0x1EC8}, {0x1ECB, 0x1ECA}, {0x1ECD, 0x1ECC}, + {0x1ECF, 0x1ECE}, {0x1ED1, 0x1ED0}, {0x1ED3, 0x1ED2}, {0x1ED5, 0x1ED4}, {0x1ED7, 0x1ED6}, {0x1ED9, 0x1ED8}, {0x1EDB, 0x1EDA}, + {0x1EDD, 0x1EDC}, {0x1EDF, 0x1EDE}, {0x1EE1, 0x1EE0}, {0x1EE3, 0x1EE2}, {0x1EE5, 0x1EE4}, {0x1EE7, 0x1EE6}, {0x1EE9, 0x1EE8}, + {0x1EEB, 0x1EEA}, {0x1EED, 0x1EEC}, {0x1EEF, 0x1EEE}, {0x1EF1, 0x1EF0}, {0x1EF3, 0x1EF2}, {0x1EF5, 0x1EF4}, {0x1EF7, 0x1EF6}, + {0x1EF9, 0x1EF8}, + // Greek Extended + {0x1F00, 0x1F08}, {0x1F01, 0x1F09}, {0x1F02, 0x1F0A}, {0x1F03, 0x1F0B}, {0x1F04, 0x1F0C}, {0x1F05, 0x1F0D}, {0x1F06, 0x1F0E}, + {0x1F07, 0x1F0F}, {0x1F10, 0x1F18}, {0x1F11, 0x1F19}, {0x1F12, 0x1F1A}, {0x1F13, 0x1F1B}, {0x1F14, 0x1F1C}, {0x1F15, 0x1F1D}, + {0x1F20, 0x1F28}, {0x1F21, 0x1F29}, {0x1F22, 0x1F2A}, {0x1F23, 0x1F2B}, {0x1F24, 0x1F2C}, {0x1F25, 0x1F2D}, {0x1F26, 0x1F2E}, + {0x1F27, 0x1F2F}, {0x1F30, 0x1F38}, {0x1F31, 0x1F39}, {0x1F32, 0x1F3A}, {0x1F33, 0x1F3B}, {0x1F34, 0x1F3C}, {0x1F35, 0x1F3D}, + {0x1F36, 0x1F3E}, {0x1F37, 0x1F3F}, {0x1F40, 0x1F48}, {0x1F41, 0x1F49}, {0x1F42, 0x1F4A}, {0x1F43, 0x1F4B}, {0x1F44, 0x1F4C}, + {0x1F45, 0x1F4D}, {0x1F51, 0x1F59}, {0x1F53, 0x1F5B}, {0x1F55, 0x1F5D}, {0x1F57, 0x1F5F}, {0x1F60, 0x1F68}, {0x1F61, 0x1F69}, + {0x1F62, 0x1F6A}, {0x1F63, 0x1F6B}, {0x1F64, 0x1F6C}, {0x1F65, 0x1F6D}, {0x1F66, 0x1F6E}, {0x1F67, 0x1F6F}, {0x1F80, 0x1F88}, + {0x1F81, 0x1F89}, {0x1F82, 0x1F8A}, {0x1F83, 0x1F8B}, {0x1F84, 0x1F8C}, {0x1F85, 0x1F8D}, {0x1F86, 0x1F8E}, {0x1F87, 0x1F8F}, + {0x1F90, 0x1F98}, {0x1F91, 0x1F99}, {0x1F92, 0x1F9A}, {0x1F93, 0x1F9B}, {0x1F94, 0x1F9C}, {0x1F95, 0x1F9D}, {0x1F96, 0x1F9E}, + {0x1F97, 0x1F9F}, {0x1FA0, 0x1FA8}, {0x1FA1, 0x1FA9}, {0x1FA2, 0x1FAA}, {0x1FA3, 0x1FAB}, {0x1FA4, 0x1FAC}, {0x1FA5, 0x1FAD}, + {0x1FA6, 0x1FAE}, {0x1FA7, 0x1FAF}, {0x1FB0, 0x1FB8}, {0x1FB1, 0x1FB9}, {0x1FD0, 0x1FD8}, {0x1FD1, 0x1FD9}, {0x1FE0, 0x1FE8}, + {0x1FE1, 0x1FE9}, + // Enclosed Alphanumerics + {0x24D0, 0x24B6}, {0x24D1, 0x24B7}, {0x24D2, 0x24B8}, {0x24D3, 0x24B9}, {0x24D4, 0x24BA}, {0x24D5, 0x24BB}, {0x24D6, 0x24BC}, + {0x24D7, 0x24BD}, {0x24D8, 0x24BE}, {0x24D9, 0x24BF}, {0x24DA, 0x24C0}, {0x24DB, 0x24C1}, {0x24DC, 0x24C2}, {0x24DD, 0x24C3}, + {0x24DE, 0x24C4}, {0x24DF, 0x24C5}, {0x24E0, 0x24C6}, {0x24E1, 0x24C7}, {0x24E2, 0x24C8}, {0x24E3, 0x24C9}, {0x24E4, 0x24CA}, + {0x24E5, 0x24CB}, {0x24E6, 0x24CC}, {0x24E7, 0x24CD}, {0x24E8, 0x24CE}, {0x24E9, 0x24CF}, + // Halfwidth and Fullwidth Forms (Latin) + {0xFF41, 0xFF21}, {0xFF42, 0xFF22}, {0xFF43, 0xFF23}, {0xFF44, 0xFF24}, {0xFF45, 0xFF25}, {0xFF46, 0xFF26}, {0xFF47, 0xFF27}, + {0xFF48, 0xFF28}, {0xFF49, 0xFF29}, {0xFF4A, 0xFF2A}, {0xFF4B, 0xFF2B}, {0xFF4C, 0xFF2C}, {0xFF4D, 0xFF2D}, {0xFF4E, 0xFF2E}, + {0xFF4F, 0xFF2F}, {0xFF50, 0xFF30}, {0xFF51, 0xFF31}, {0xFF52, 0xFF32}, {0xFF53, 0xFF33}, {0xFF54, 0xFF34}, {0xFF55, 0xFF35}, + {0xFF56, 0xFF36}, {0xFF57, 0xFF37}, {0xFF58, 0xFF38}, {0xFF59, 0xFF39}, {0xFF5A, 0xFF3A} + // clang-format on +}; -using std::vector; -static const vector low_dataset = { - 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, - 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, - 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F8, 0x00F9, - 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF, 0x0101, 0x0103, 0x0105, 0x0107, 0x0109, 0x010B, 0x010D, 0x010F, 0x0111, 0x0113, 0x0115, - 0x0117, 0x0119, 0x011B, 0x011D, 0x011F, 0x0121, 0x0123, 0x0125, 0x0127, 0x0129, 0x012B, 0x012D, 0x012F, 0x0131, 0x0133, 0x0135, 0x0137, - 0x013A, 0x013C, 0x013E, 0x0140, 0x0142, 0x0144, 0x0146, 0x0148, 0x014B, 0x014D, 0x014F, 0x0151, 0x0153, 0x0155, 0x0157, 0x0159, 0x015B, - 0x015D, 0x015F, 0x0161, 0x0163, 0x0165, 0x0167, 0x0169, 0x016B, 0x016D, 0x016F, 0x0171, 0x0173, 0x0175, 0x0177, 0x017A, 0x017C, 0x017E, - 0x0183, 0x0185, 0x0188, 0x018C, 0x0192, 0x0199, 0x01A1, 0x01A3, 0x01A5, 0x01A8, 0x01AD, 0x01B0, 0x01B4, 0x01B6, 0x01B9, 0x01BD, 0x01C6, - 0x01C9, 0x01CC, 0x01CE, 0x01D0, 0x01D2, 0x01D4, 0x01D6, 0x01D8, 0x01DA, 0x01DC, 0x01DF, 0x01E1, 0x01E3, 0x01E5, 0x01E7, 0x01E9, 0x01EB, - 0x01ED, 0x01EF, 0x01F3, 0x01F5, 0x01FB, 0x01FD, 0x01FF, 0x0201, 0x0203, 0x0205, 0x0207, 0x0209, 0x020B, 0x020D, 0x020F, 0x0211, 0x0213, - 0x0215, 0x0217, 0x0253, 0x0254, 0x0257, 0x0258, 0x0259, 0x025B, 0x0260, 0x0263, 0x0268, 0x0269, 0x026F, 0x0272, 0x0275, 0x0283, 0x0288, - 0x028A, 0x028B, 0x0292, 0x03AC, 0x03AD, 0x03AE, 0x03AF, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, - 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, - 0x03CD, 0x03CE, 0x03E3, 0x03E5, 0x03E7, 0x03E9, 0x03EB, 0x03ED, 0x03EF, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, - 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045A, - 0x045B, 0x045C, 0x045E, 0x045F, 0x0461, 0x0463, 0x0465, 0x0467, 0x0469, 0x046B, 0x046D, 0x046F, 0x0471, 0x0473, 0x0475, 0x0477, 0x0479, - 0x047B, 0x047D, 0x047F, 0x0481, 0x0491, 0x0493, 0x0495, 0x0497, 0x0499, 0x049B, 0x049D, 0x049F, 0x04A1, 0x04A3, 0x04A5, 0x04A7, 0x04A9, - 0x04AB, 0x04AD, 0x04AF, 0x04B1, 0x04B3, 0x04B5, 0x04B7, 0x04B9, 0x04BB, 0x04BD, 0x04BF, 0x04C2, 0x04C4, 0x04C8, 0x04CC, 0x04D1, 0x04D3, - 0x04D5, 0x04D7, 0x04D9, 0x04DB, 0x04DD, 0x04DF, 0x04E1, 0x04E3, 0x04E5, 0x04E7, 0x04E9, 0x04EB, 0x04EF, 0x04F1, 0x04F3, 0x04F5, 0x04F9, - 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, 0x0568, 0x0569, 0x056A, 0x056B, 0x056C, 0x056D, 0x056E, 0x056F, 0x0570, 0x0571, - 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, 0x0578, 0x0579, 0x057A, 0x057B, 0x057C, 0x057D, 0x057E, 0x057F, 0x0580, 0x0581, 0x0582, - 0x0583, 0x0584, 0x0585, 0x0586, 0x10D0, 0x10D1, 0x10D2, 0x10D3, 0x10D4, 0x10D5, 0x10D6, 0x10D7, 0x10D8, 0x10D9, 0x10DA, 0x10DB, 0x10DC, - 0x10DD, 0x10DE, 0x10DF, 0x10E0, 0x10E1, 0x10E2, 0x10E3, 0x10E4, 0x10E5, 0x10E6, 0x10E7, 0x10E8, 0x10E9, 0x10EA, 0x10EB, 0x10EC, 0x10ED, - 0x10EE, 0x10EF, 0x10F0, 0x10F1, 0x10F2, 0x10F3, 0x10F4, 0x10F5, 0x1E01, 0x1E03, 0x1E05, 0x1E07, 0x1E09, 0x1E0B, 0x1E0D, 0x1E0F, 0x1E11, - 0x1E13, 0x1E15, 0x1E17, 0x1E19, 0x1E1B, 0x1E1D, 0x1E1F, 0x1E21, 0x1E23, 0x1E25, 0x1E27, 0x1E29, 0x1E2B, 0x1E2D, 0x1E2F, 0x1E31, 0x1E33, - 0x1E35, 0x1E37, 0x1E39, 0x1E3B, 0x1E3D, 0x1E3F, 0x1E41, 0x1E43, 0x1E45, 0x1E47, 0x1E49, 0x1E4B, 0x1E4D, 0x1E4F, 0x1E51, 0x1E53, 0x1E55, - 0x1E57, 0x1E59, 0x1E5B, 0x1E5D, 0x1E5F, 0x1E61, 0x1E63, 0x1E65, 0x1E67, 0x1E69, 0x1E6B, 0x1E6D, 0x1E6F, 0x1E71, 0x1E73, 0x1E75, 0x1E77, - 0x1E79, 0x1E7B, 0x1E7D, 0x1E7F, 0x1E81, 0x1E83, 0x1E85, 0x1E87, 0x1E89, 0x1E8B, 0x1E8D, 0x1E8F, 0x1E91, 0x1E93, 0x1E95, 0x1EA1, 0x1EA3, - 0x1EA5, 0x1EA7, 0x1EA9, 0x1EAB, 0x1EAD, 0x1EAF, 0x1EB1, 0x1EB3, 0x1EB5, 0x1EB7, 0x1EB9, 0x1EBB, 0x1EBD, 0x1EBF, 0x1EC1, 0x1EC3, 0x1EC5, - 0x1EC7, 0x1EC9, 0x1ECB, 0x1ECD, 0x1ECF, 0x1ED1, 0x1ED3, 0x1ED5, 0x1ED7, 0x1ED9, 0x1EDB, 0x1EDD, 0x1EDF, 0x1EE1, 0x1EE3, 0x1EE5, 0x1EE7, - 0x1EE9, 0x1EEB, 0x1EED, 0x1EEF, 0x1EF1, 0x1EF3, 0x1EF5, 0x1EF7, 0x1EF9, 0x1F00, 0x1F01, 0x1F02, 0x1F03, 0x1F04, 0x1F05, 0x1F06, 0x1F07, - 0x1F10, 0x1F11, 0x1F12, 0x1F13, 0x1F14, 0x1F15, 0x1F20, 0x1F21, 0x1F22, 0x1F23, 0x1F24, 0x1F25, 0x1F26, 0x1F27, 0x1F30, 0x1F31, 0x1F32, - 0x1F33, 0x1F34, 0x1F35, 0x1F36, 0x1F37, 0x1F40, 0x1F41, 0x1F42, 0x1F43, 0x1F44, 0x1F45, 0x1F51, 0x1F53, 0x1F55, 0x1F57, 0x1F60, 0x1F61, - 0x1F62, 0x1F63, 0x1F64, 0x1F65, 0x1F66, 0x1F67, 0x1F80, 0x1F81, 0x1F82, 0x1F83, 0x1F84, 0x1F85, 0x1F86, 0x1F87, 0x1F90, 0x1F91, 0x1F92, - 0x1F93, 0x1F94, 0x1F95, 0x1F96, 0x1F97, 0x1FA0, 0x1FA1, 0x1FA2, 0x1FA3, 0x1FA4, 0x1FA5, 0x1FA6, 0x1FA7, 0x1FB0, 0x1FB1, 0x1FD0, 0x1FD1, - 0x1FE0, 0x1FE1, 0x24D0, 0x24D1, 0x24D2, 0x24D3, 0x24D4, 0x24D5, 0x24D6, 0x24D7, 0x24D8, 0x24D9, 0x24DA, 0x24DB, 0x24DC, 0x24DD, 0x24DE, - 0x24DF, 0x24E0, 0x24E1, 0x24E2, 0x24E3, 0x24E4, 0x24E5, 0x24E6, 0x24E7, 0x24E8, 0x24E9, 0xFF41, 0xFF42, 0xFF43, 0xFF44, 0xFF45, 0xFF46, - 0xFF47, 0xFF48, 0xFF49, 0xFF4A, 0xFF4B, 0xFF4C, 0xFF4D, 0xFF4E, 0xFF4F, 0xFF50, 0xFF51, 0xFF52, 0xFF53, 0xFF54, 0xFF55, 0xFF56, 0xFF57, - 0xFF58, 0xFF59, 0xFF5A}; - -static const vector up_dataset = { - 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050, 0x0051, - 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005A, 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, - 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D8, 0x00D9, - 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x0178, 0x0100, 0x0102, 0x0104, 0x0106, 0x0108, 0x010A, 0x010C, 0x010E, 0x0110, 0x0112, 0x0114, - 0x0116, 0x0118, 0x011A, 0x011C, 0x011E, 0x0120, 0x0122, 0x0124, 0x0126, 0x0128, 0x012A, 0x012C, 0x012E, 0x0049, 0x0132, 0x0134, 0x0136, - 0x0139, 0x013B, 0x013D, 0x013F, 0x0141, 0x0143, 0x0145, 0x0147, 0x014A, 0x014C, 0x014E, 0x0150, 0x0152, 0x0154, 0x0156, 0x0158, 0x015A, - 0x015C, 0x015E, 0x0160, 0x0162, 0x0164, 0x0166, 0x0168, 0x016A, 0x016C, 0x016E, 0x0170, 0x0172, 0x0174, 0x0176, 0x0179, 0x017B, 0x017D, - 0x0182, 0x0184, 0x0187, 0x018B, 0x0191, 0x0198, 0x01A0, 0x01A2, 0x01A4, 0x01A7, 0x01AC, 0x01AF, 0x01B3, 0x01B5, 0x01B8, 0x01BC, 0x01C4, - 0x01C7, 0x01CA, 0x01CD, 0x01CF, 0x01D1, 0x01D3, 0x01D5, 0x01D7, 0x01D9, 0x01DB, 0x01DE, 0x01E0, 0x01E2, 0x01E4, 0x01E6, 0x01E8, 0x01EA, - 0x01EC, 0x01EE, 0x01F1, 0x01F4, 0x01FA, 0x01FC, 0x01FE, 0x0200, 0x0202, 0x0204, 0x0206, 0x0208, 0x020A, 0x020C, 0x020E, 0x0210, 0x0212, - 0x0214, 0x0216, 0x0181, 0x0186, 0x018A, 0x018E, 0x018F, 0x0190, 0x0193, 0x0194, 0x0197, 0x0196, 0x019C, 0x019D, 0x019F, 0x01A9, 0x01AE, - 0x01B1, 0x01B2, 0x01B7, 0x0386, 0x0388, 0x0389, 0x038A, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, - 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x038C, - 0x038E, 0x038F, 0x03E2, 0x03E4, 0x03E6, 0x03E8, 0x03EA, 0x03EC, 0x03EE, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, - 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, - 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, 0x040A, - 0x040B, 0x040C, 0x040E, 0x040F, 0x0460, 0x0462, 0x0464, 0x0466, 0x0468, 0x046A, 0x046C, 0x046E, 0x0470, 0x0472, 0x0474, 0x0476, 0x0478, - 0x047A, 0x047C, 0x047E, 0x0480, 0x0490, 0x0492, 0x0494, 0x0496, 0x0498, 0x049A, 0x049C, 0x049E, 0x04A0, 0x04A2, 0x04A4, 0x04A6, 0x04A8, - 0x04AA, 0x04AC, 0x04AE, 0x04B0, 0x04B2, 0x04B4, 0x04B6, 0x04B8, 0x04BA, 0x04BC, 0x04BE, 0x04C1, 0x04C3, 0x04C7, 0x04CB, 0x04D0, 0x04D2, - 0x04D4, 0x04D6, 0x04D8, 0x04DA, 0x04DC, 0x04DE, 0x04E0, 0x04E2, 0x04E4, 0x04E6, 0x04E8, 0x04EA, 0x04EE, 0x04F0, 0x04F2, 0x04F4, 0x04F8, - 0x0531, 0x0532, 0x0533, 0x0534, 0x0535, 0x0536, 0x0537, 0x0538, 0x0539, 0x053A, 0x053B, 0x053C, 0x053D, 0x053E, 0x053F, 0x0540, 0x0541, - 0x0542, 0x0543, 0x0544, 0x0545, 0x0546, 0x0547, 0x0548, 0x0549, 0x054A, 0x054B, 0x054C, 0x054D, 0x054E, 0x054F, 0x0550, 0x0551, 0x0552, - 0x0553, 0x0554, 0x0555, 0x0556, 0x10A0, 0x10A1, 0x10A2, 0x10A3, 0x10A4, 0x10A5, 0x10A6, 0x10A7, 0x10A8, 0x10A9, 0x10AA, 0x10AB, 0x10AC, - 0x10AD, 0x10AE, 0x10AF, 0x10B0, 0x10B1, 0x10B2, 0x10B3, 0x10B4, 0x10B5, 0x10B6, 0x10B7, 0x10B8, 0x10B9, 0x10BA, 0x10BB, 0x10BC, 0x10BD, - 0x10BE, 0x10BF, 0x10C0, 0x10C1, 0x10C2, 0x10C3, 0x10C4, 0x10C5, 0x1E00, 0x1E02, 0x1E04, 0x1E06, 0x1E08, 0x1E0A, 0x1E0C, 0x1E0E, 0x1E10, - 0x1E12, 0x1E14, 0x1E16, 0x1E18, 0x1E1A, 0x1E1C, 0x1E1E, 0x1E20, 0x1E22, 0x1E24, 0x1E26, 0x1E28, 0x1E2A, 0x1E2C, 0x1E2E, 0x1E30, 0x1E32, - 0x1E34, 0x1E36, 0x1E38, 0x1E3A, 0x1E3C, 0x1E3E, 0x1E40, 0x1E42, 0x1E44, 0x1E46, 0x1E48, 0x1E4A, 0x1E4C, 0x1E4E, 0x1E50, 0x1E52, 0x1E54, - 0x1E56, 0x1E58, 0x1E5A, 0x1E5C, 0x1E5E, 0x1E60, 0x1E62, 0x1E64, 0x1E66, 0x1E68, 0x1E6A, 0x1E6C, 0x1E6E, 0x1E70, 0x1E72, 0x1E74, 0x1E76, - 0x1E78, 0x1E7A, 0x1E7C, 0x1E7E, 0x1E80, 0x1E82, 0x1E84, 0x1E86, 0x1E88, 0x1E8A, 0x1E8C, 0x1E8E, 0x1E90, 0x1E92, 0x1E94, 0x1EA0, 0x1EA2, - 0x1EA4, 0x1EA6, 0x1EA8, 0x1EAA, 0x1EAC, 0x1EAE, 0x1EB0, 0x1EB2, 0x1EB4, 0x1EB6, 0x1EB8, 0x1EBA, 0x1EBC, 0x1EBE, 0x1EC0, 0x1EC2, 0x1EC4, - 0x1EC6, 0x1EC8, 0x1ECA, 0x1ECC, 0x1ECE, 0x1ED0, 0x1ED2, 0x1ED4, 0x1ED6, 0x1ED8, 0x1EDA, 0x1EDC, 0x1EDE, 0x1EE0, 0x1EE2, 0x1EE4, 0x1EE6, - 0x1EE8, 0x1EEA, 0x1EEC, 0x1EEE, 0x1EF0, 0x1EF2, 0x1EF4, 0x1EF6, 0x1EF8, 0x1F08, 0x1F09, 0x1F0A, 0x1F0B, 0x1F0C, 0x1F0D, 0x1F0E, 0x1F0F, - 0x1F18, 0x1F19, 0x1F1A, 0x1F1B, 0x1F1C, 0x1F1D, 0x1F28, 0x1F29, 0x1F2A, 0x1F2B, 0x1F2C, 0x1F2D, 0x1F2E, 0x1F2F, 0x1F38, 0x1F39, 0x1F3A, - 0x1F3B, 0x1F3C, 0x1F3D, 0x1F3E, 0x1F3F, 0x1F48, 0x1F49, 0x1F4A, 0x1F4B, 0x1F4C, 0x1F4D, 0x1F59, 0x1F5B, 0x1F5D, 0x1F5F, 0x1F68, 0x1F69, - 0x1F6A, 0x1F6B, 0x1F6C, 0x1F6D, 0x1F6E, 0x1F6F, 0x1F88, 0x1F89, 0x1F8A, 0x1F8B, 0x1F8C, 0x1F8D, 0x1F8E, 0x1F8F, 0x1F98, 0x1F99, 0x1F9A, - 0x1F9B, 0x1F9C, 0x1F9D, 0x1F9E, 0x1F9F, 0x1FA8, 0x1FA9, 0x1FAA, 0x1FAB, 0x1FAC, 0x1FAD, 0x1FAE, 0x1FAF, 0x1FB8, 0x1FB9, 0x1FD8, 0x1FD9, - 0x1FE8, 0x1FE9, 0x24B6, 0x24B7, 0x24B8, 0x24B9, 0x24BA, 0x24BB, 0x24BC, 0x24BD, 0x24BE, 0x24BF, 0x24C0, 0x24C1, 0x24C2, 0x24C3, 0x24C4, - 0x24C5, 0x24C6, 0x24C7, 0x24C8, 0x24C9, 0x24CA, 0x24CB, 0x24CC, 0x24CD, 0x24CE, 0x24CF, 0xFF21, 0xFF22, 0xFF23, 0xFF24, 0xFF25, 0xFF26, - 0xFF27, 0xFF28, 0xFF29, 0xFF2A, 0xFF2B, 0xFF2C, 0xFF2D, 0xFF2E, 0xFF2F, 0xFF30, 0xFF31, 0xFF32, 0xFF33, 0xFF34, 0xFF35, 0xFF36, 0xFF37, - 0xFF38, 0xFF39, 0xFF3A}; - -using std::vector; +constexpr int checkAlphabetSorted() { + for (int i = 1; i < std::end(kAlphabet) - std::begin(kAlphabet); i++) { + if (kAlphabet[i - 1] >= kAlphabet[i]) return i; + } + return -1; +} +static_assert(checkAlphabetSorted() == -1, "Alphabet must be sorted"); -class CustomLocal { +class CustomLocale { public: - static CustomLocal& Init() noexcept { - static CustomLocal singleton; - return singleton; +#ifndef _WIN32 + constexpr +#endif // _WIN32 + CustomLocale() noexcept { + for (uint32_t i = 0; i < UINT16_MAX; ++i) { + customLocale_[i].isAlpha = false; + customLocale_[i].lower = uint16_t(i); + } + for (const auto& a : kAlphabet) { + customLocale_[a.first].isAlpha = true; + customLocale_[a.first].lower = a.first; + if (!customLocale_[a.second].isAlpha) { + customLocale_[a.second].isAlpha = true; + customLocale_[a.second].lower = a.first; + } + } } - void ToLower(wstring& data) noexcept { - for (size_t i = 0; i < data.size(); ++i) { - if (data[i] < UINT16_MAX && data[i] > 0) { - data[i] = custom_local_[data[i]].lower; + void ToLower(std::wstring& data) const noexcept { + for (auto& d : data) { + if (d < UINT16_MAX && d > 0) { + d = customLocale_[d].lower; } } } - wchar_t ToLower(wchar_t ch) noexcept { + wchar_t ToLower(wchar_t ch) const noexcept { uint32_t ofs = ch; if (ofs < UINT16_MAX) { - ch = custom_local_[ofs].lower; + ch = customLocale_[ofs].lower; } return ch; } - bool IsAlpha(wchar_t ch) noexcept { + bool IsAlpha(wchar_t ch) const noexcept { uint32_t ofs = ch; - if (ofs >= custom_local_.size()) { + if (ofs >= UINT16_MAX) { return false; } - return custom_local_[ofs].is_alfa; + return customLocale_[ofs].isAlpha; } private: - CustomLocal() { - for (uint32_t i = 0; i < custom_local_.size(); ++i) { - auto it = std::find(up_dataset.begin(), up_dataset.end(), i); - if (it != up_dataset.end()) { - custom_local_[i].lower = *(low_dataset.begin() + (it - up_dataset.begin())); - custom_local_[i].is_alfa = true; - } else { - custom_local_[i].lower = uint16_t(i); - it = std::find(low_dataset.begin(), low_dataset.end(), i); - custom_local_[i].is_alfa = (it != low_dataset.end()); - } - } - } - CustomLocal(const CustomLocal&) = delete; - CustomLocal& operator=(const CustomLocal&) = delete; + CustomLocale(const CustomLocale&) = delete; + CustomLocale& operator=(const CustomLocale&) = delete; struct LocalCtx { + constexpr LocalCtx() noexcept : lower(0), isAlpha(false) {} + uint16_t lower; - bool is_alfa; + bool isAlpha; }; - std::array custom_local_; + std::array customLocale_; }; -void ToLower(wstring& data) noexcept { CustomLocal::Init().ToLower(data); } -wchar_t ToLower(wchar_t ch) noexcept { return CustomLocal::Init().ToLower(ch); } +#ifdef _WIN32 +static const CustomLocal kCustomLocale; +#else // _WIN32 +constexpr CustomLocale kCustomLocale; +#endif // _WIN32 + +void ToLower(std::wstring& data) noexcept { kCustomLocale.ToLower(data); } +wchar_t ToLower(wchar_t ch) noexcept { return kCustomLocale.ToLower(ch); } + +bool IsAlpha(wchar_t ch) noexcept { return kCustomLocale.IsAlpha(ch); } -bool IsAlpha(wchar_t ch) noexcept { return CustomLocal::Init().IsAlpha(ch); } -bool IsDigit(wchar_t ch) noexcept { return ch >= '0' && ch <= '9'; } } // namespace reindexer diff --git a/cpp_src/tools/customlocal.h b/cpp_src/tools/customlocal.h index 120607b88..e16d765f1 100644 --- a/cpp_src/tools/customlocal.h +++ b/cpp_src/tools/customlocal.h @@ -3,11 +3,10 @@ namespace reindexer { -using std::wstring; - -void ToLower(wstring& data) noexcept; +void ToLower(std::wstring& data) noexcept; wchar_t ToLower(wchar_t ch) noexcept; bool IsAlpha(wchar_t ch) noexcept; -bool IsDigit(wchar_t ch) noexcept; +inline bool IsDigit(wchar_t ch) noexcept { return ch >= '0' && ch <= '9'; } + } // namespace reindexer diff --git a/cpp_src/tools/logger.cc b/cpp_src/tools/logger.cc index 7d7f11efb..4c0700d78 100644 --- a/cpp_src/tools/logger.cc +++ b/cpp_src/tools/logger.cc @@ -1,36 +1,66 @@ #include "tools/logger.h" -#include +#include +#include "estl/mutex.h" #include "estl/shared_mutex.h" -#include "estl/smart_lock.h" namespace reindexer { -LogWriter g_logWriter; -shared_timed_mutex g_LoggerLock; -std::atomic_bool g_MtLogger = {true}; +static LogWriter g_logWriter; +static read_write_spinlock g_LoggerMtx; +static std::atomic g_MtLogger = {LoggerPolicy::NotInit}; -void write(int level, char *buf) { - if (!g_logWriter) return; - g_logWriter(level, buf); +RX_ALWAYS_INLINE void write(int level, char *buf) { + if (g_logWriter) { + g_logWriter(level, buf); + } } void logPrint(int level, char *buf) { - if (g_MtLogger) { - smart_lock lk(g_LoggerLock, false); - write(level, buf); - } else { - write(level, buf); + switch (g_MtLogger.load(std::memory_order_relaxed)) { + case LoggerPolicy::NotInit: + case LoggerPolicy::WithLocks: { + shared_lock lck(g_LoggerMtx); + write(level, buf); + } break; + case LoggerPolicy::WithoutLocks: + write(level, buf); + break; } } -void logInstallWriter(LogWriter writer, bool multithreaded) { - if (g_MtLogger || multithreaded) { - smart_lock lk(g_LoggerLock, true); +void logInstallWriter(LogWriter writer, LoggerPolicy policy) { + std::string errorText; + + static std::mutex g_LoggerPolicyMtx; + std::unique_lock lck(g_LoggerPolicyMtx); + + const auto curPolicy = g_MtLogger.load(std::memory_order_relaxed); + if (curPolicy != LoggerPolicy::NotInit && policy != curPolicy) { + errorText = + fmt::sprintf("Attempt to switch logger's lock policy, which was previously set. Current: %d; new: %d. Logger was not changed", + int(curPolicy), int(policy)); + fputs(errorText.c_str(), stderr); + fputs("\n", stderr); + fflush(stderr); +#if defined(REINDEX_WITH_ASAN) || defined(REINDEX_WITH_TSAN) || defined(RX_WITH_STDLIB_DEBUG) + std::abort(); +#else + lck.unlock(); + if (writer) { + errorText.append(". THIS logger is not active"); + writer(LogError, &errorText[0]); + } + return; +#endif + } + + g_MtLogger.store(policy, std::memory_order_relaxed); + + if (curPolicy == LoggerPolicy::WithLocks || policy == LoggerPolicy::WithLocks) { + std::lock_guard logLck(g_LoggerMtx); g_logWriter = std::move(writer); - g_MtLogger = multithreaded; } else { g_logWriter = std::move(writer); - g_MtLogger = multithreaded; } } diff --git a/cpp_src/tools/logger.h b/cpp_src/tools/logger.h index 807d14ee3..6597ae487 100644 --- a/cpp_src/tools/logger.h +++ b/cpp_src/tools/logger.h @@ -15,5 +15,7 @@ void logPrintf(int level, const char* fmt, const Args&... args) { logPrint(level, &str[0]); } -void logInstallWriter(LogWriter writer, bool multithreaded = true); +enum class LoggerPolicy : int { NotInit, WithLocks, WithoutLocks }; + +void logInstallWriter(LogWriter writer, LoggerPolicy mode); } // namespace reindexer diff --git a/cpp_src/tools/logginglongqueries.cc b/cpp_src/tools/logginglongqueries.cc index 0ecbbd581..858c9ea61 100644 --- a/cpp_src/tools/logginglongqueries.cc +++ b/cpp_src/tools/logginglongqueries.cc @@ -1,6 +1,5 @@ #include "logginglongqueries.h" #include "core/nsselecter/explaincalc.h" -#include "core/query/query.h" #include "core/transactionimpl.h" #include "logger.h" @@ -69,8 +68,21 @@ static std::string_view describeDurationStorageIdx(DurationStorageIdx idx) { throw Error(errLogic, "Unknown duration storage index"); } +template +static auto fillStorageInfo(std::ostringstream& os, const Storage& storage) { + os << "[slowlog] Waiting for a mutex lock:" << std::endl; + for (size_t i = 0; i < storage.size(); ++i) { + if (!storage.at(i)) { + continue; + } + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + os << describeDurationStorageIdx(DurationStorageIdx{static_cast(i)}) << ": " << storage.at(i)->count() << "us" + << std::endl; + } +} + template <> -void Logger::Dump(std::chrono::microseconds time) { +void Logger>::Dump(std::chrono::microseconds time) { if (wrapper_.loggingParams.thresholdUs >= 0 && time.count() > wrapper_.loggingParams.thresholdUs) { std::ostringstream os; os << fmt::sprintf("[slowlog] Long execution query: sql - %s; (%dus)\n", wrapper_.query.GetSQL(wrapper_.loggingParams.normalized), @@ -83,7 +95,27 @@ void Logger::Dump(std::chrono::microseconds time) { os << describeExplainDuration(ExplainDuration{i}) << ": " << (*wrapper_.durationStorage)[i].count() << "us" << std::endl; } } + logPrint(LogWarning, os.str().data()); + } +} +template <> +void Logger>::Dump(std::chrono::microseconds time) { + if (wrapper_.loggingParams.thresholdUs >= 0 && time.count() > wrapper_.loggingParams.thresholdUs) { + std::ostringstream os; + os << fmt::sprintf("[slowlog] Long execution query: sql - %s; (%dus)\n", wrapper_.query.GetSQL(wrapper_.loggingParams.normalized), + time.count()); + fillStorageInfo(os, wrapper_.durationStorage); + logPrint(LogWarning, os.str().data()); + } +} +template <> +void Logger>::Dump(std::chrono::microseconds time) { + if (wrapper_.loggingParams.thresholdUs >= 0 && time.count() > wrapper_.loggingParams.thresholdUs) { + std::ostringstream os; + os << fmt::sprintf("[slowlog] Long execution query: sql - %s; (%dus)\n", wrapper_.query.GetSQL(wrapper_.loggingParams.normalized), + time.count()); + fillStorageInfo(os, wrapper_.durationStorage); logPrint(LogWarning, os.str().data()); } } @@ -101,13 +133,7 @@ void Logger::Dump(std::chrono::microseconds time) { if (longAvgStep || longTotal) { std::ostringstream os; - os << "[slowlog] Waiting for a mutex lock:" << std::endl; - for (size_t i = 0; i < wrapper_.durationStorage.size(); ++i) { - if (wrapper_.durationStorage[i]) { - os << describeDurationStorageIdx(DurationStorageIdx{static_cast(i)}) << ": " - << wrapper_.durationStorage[i]->count() << "us" << std::endl; // NOLINT(bugprone-unchecked-optional-access) - } - } + fillStorageInfo(os, wrapper_.durationStorage); logPrintf(LogWarning, "[slowlog] Long tx apply: namespace - %s; was%scopied; %d steps;%s%s\n%s", wrapper_.tx.GetName(), wrapper_.wasCopied ? " " : " not ", wrapper_.tx.GetSteps().size(), @@ -116,16 +142,18 @@ void Logger::Dump(std::chrono::microseconds time) { } } -template ::ExplainMethodType... methods> -void ActionWrapper::add(const ExplainCalc& explain) { +template >::ExplainMethodType... methods> +void ActionWrapper>::add(const ExplainCalc& explain) { durationStorage = {std::chrono::duration_cast((explain.*methods)())...}; } -void ActionWrapper::Add(const ExplainCalc& explain) { +void ActionWrapper>::Add(const ExplainCalc& explain) { add<&ExplainCalc::Total, &ExplainCalc::Prepare, &ExplainCalc::Indexes, &ExplainCalc::Postprocess, &ExplainCalc::Loop, &ExplainCalc::Sort>(explain); } -template struct Logger; template struct Logger; +template struct Logger>; +template struct Logger>; +template struct Logger>; } // namespace reindexer::long_actions diff --git a/cpp_src/tools/logginglongqueries.h b/cpp_src/tools/logginglongqueries.h index d79bcd2ad..0a4b42f2e 100644 --- a/cpp_src/tools/logginglongqueries.h +++ b/cpp_src/tools/logginglongqueries.h @@ -8,7 +8,6 @@ namespace reindexer { class Query; class Transaction; - class ExplainCalc; namespace long_actions { @@ -48,21 +47,6 @@ constexpr DurationStorageIdx DurationStorageIdxCast(MutexMark mark) { } } -template <> -struct ActionWrapper { - const Transaction& tx; - LongTxLoggingParams thresholds; - const bool& wasCopied; - - using ArrayT = std::array, size_t(DurationStorageIdx::StorageSize)>; - ArrayT durationStorage = {}; - - void Add(DurationStorageIdx idx, std::chrono::microseconds time) { - auto& val = durationStorage[static_cast(idx)]; - val ? * val += time : val = time; - } -}; - enum class ExplainDuration { Total, Prepare, @@ -73,10 +57,40 @@ enum class ExplainDuration { ExplainDurationSize, }; -template <> -struct ActionWrapper { +template +struct QueryEnum2Type : std::integral_constant {}; + +struct LockDurationStorage { + using ArrayT = std::array, size_t(DurationStorageIdx::StorageSize)>; + ArrayT durationStorage = {}; + void Add(DurationStorageIdx idx, std::chrono::microseconds time) { + auto& val = durationStorage[static_cast(idx)]; + val ? * val += time : val = time; + } +}; + +struct QueryParams { const Query& query; LongQueriesLoggingParams loggingParams; +}; + +struct TransactionParams { + const Transaction& tx; + LongTxLoggingParams thresholds; + const bool& wasCopied; +}; + +template <> +struct ActionWrapper : TransactionParams, LockDurationStorage { + template + ActionWrapper(Args&&... args) : TransactionParams{std::forward(args)...}, LockDurationStorage() {} +}; + +template <> +struct ActionWrapper> : QueryParams { + template + ActionWrapper(Args&&... args) : QueryParams{std::forward(args)...} {} + using ArrayT = std::array; std::optional durationStorage = std::nullopt; @@ -88,11 +102,22 @@ struct ActionWrapper { void add(const ExplainCalc&); }; -template +template <> +struct ActionWrapper> : QueryParams, LockDurationStorage { + template + ActionWrapper(Args&&... args) : QueryParams{std::forward(args)...}, LockDurationStorage{} {} +}; + +template <> +struct ActionWrapper> : QueryParams, LockDurationStorage { + template + ActionWrapper(Args&&... args) : QueryParams{std::forward(args)...}, LockDurationStorage{} {} +}; + +template struct Logger { + Logger() = default; static constexpr bool isEnabled = !std::is_empty_v>; - template - Logger(const ActionType& action, Args&&... args) : wrapper_(ActionWrapper{action, std::forward(args)...}) {} void Dump(std::chrono::microseconds time); template @@ -101,15 +126,40 @@ struct Logger { } private: + template + friend auto MakeLogger(Args&&... args); + + template + friend auto MakeLogger(Args&&... args); + + template + Logger(const ActionType& action, Args&&... args) : wrapper_(ActionWrapper{action, std::forward(args)...}) {} ActionWrapper wrapper_; }; +template +auto MakeLogger(Args&&... args) { + return Logger{std::forward(args)...}; +} + +template +auto MakeLogger(Args&&... args) { + return Logger>{std::forward(args)...}; +} + template <> -void Logger::Dump(std::chrono::microseconds); +void Logger>::Dump(std::chrono::microseconds); +template <> +void Logger>::Dump(std::chrono::microseconds); +template <> +void Logger>::Dump(std::chrono::microseconds); template <> void Logger::Dump(std::chrono::microseconds); -extern template struct Logger; extern template struct Logger; +extern template struct Logger>; +extern template struct Logger>; +extern template struct Logger>; + } // namespace long_actions } // namespace reindexer \ No newline at end of file diff --git a/cpp_src/tools/md5crypt.cc b/cpp_src/tools/md5crypt.cc index 86fa290d6..384ab8e13 100644 --- a/cpp_src/tools/md5crypt.cc +++ b/cpp_src/tools/md5crypt.cc @@ -14,7 +14,7 @@ namespace reindexer { -const std::string kMD5CryptDelimiter = "$"; +constexpr std::string_view kMD5CryptDelimiter = "$"; std::string MD5crypt(const std::string &passwd, const std::string &salt) noexcept { static unsigned const char cov2char[64] = { @@ -32,9 +32,9 @@ std::string MD5crypt(const std::string &passwd, const std::string &salt) noexcep { MD5 md; md.add(passwd.c_str(), passwd.size()); - md.add(kMD5CryptDelimiter.c_str(), kMD5CryptDelimiter.size()); + md.add(kMD5CryptDelimiter.data(), kMD5CryptDelimiter.size()); md.add(reindexer::kMD5CryptMagic.data(), reindexer::kMD5CryptMagic.size()); - md.add(kMD5CryptDelimiter.c_str(), kMD5CryptDelimiter.size()); + md.add(kMD5CryptDelimiter.data(), kMD5CryptDelimiter.size()); md.add(trunkatedSalt, saltLen); MD5 md2; diff --git a/cpp_src/tools/stringstools.cc b/cpp_src/tools/stringstools.cc index 2b7280ea0..ab9cfa83d 100644 --- a/cpp_src/tools/stringstools.cc +++ b/cpp_src/tools/stringstools.cc @@ -52,25 +52,25 @@ std::string unescapeString(std::string_view str) { return dst; } -wstring &utf8_to_utf16(std::string_view src, wstring &dst) { +std::wstring &utf8_to_utf16(std::string_view src, std::wstring &dst) { dst.resize(src.length()); auto end = utf8::unchecked::utf8to32(src.begin(), src.end(), dst.begin()); dst.resize(std::distance(dst.begin(), end)); return dst; } -std::string &utf16_to_utf8(const wstring &src, std::string &dst) { +std::string &utf16_to_utf8(const std::wstring &src, std::string &dst) { dst.resize(src.length() * 4); auto end = utf8::unchecked::utf32to8(src.begin(), src.end(), dst.begin()); dst.resize(std::distance(dst.begin(), end)); return dst; } -wstring utf8_to_utf16(std::string_view src) { - wstring dst; +std::wstring utf8_to_utf16(std::string_view src) { + std::wstring dst; return utf8_to_utf16(src, dst); } -std::string utf16_to_utf8(const wstring &src) { +std::string utf16_to_utf8(const std::wstring &src) { std::string dst; return utf16_to_utf8(src, dst); } @@ -147,6 +147,7 @@ bool is_number(std::string_view str) { } void split(std::string_view str, std::string &buf, std::vector &words, const std::string &extraWordSymbols) { + //assuming that the 'ToLower' function and the 'check for replacement' function should not change the character size in bytes buf.resize(str.length()); words.resize(0); auto bufIt = buf.begin(); @@ -300,7 +301,7 @@ std::pair Word2PosHelper::convert(int wordPos, int endPos) { return ret; } -void split(std::string_view utf8Str, wstring &utf16str, std::vector &words, const std::string &extraWordSymbols) { +void split(std::string_view utf8Str, std::wstring &utf16str, std::vector &words, const std::string &extraWordSymbols) { utf8_to_utf16(utf8Str, utf16str); words.resize(0); for (auto it = utf16str.begin(); it != utf16str.end();) { @@ -309,7 +310,7 @@ void split(std::string_view utf8Str, wstring &utf16str, std::vector bool checkIfStartsWith(std::string_view pattern, std::string_view str) noexcept { if (pattern.empty() || str.empty()) return false; diff --git a/cpp_src/tools/stringstools.h b/cpp_src/tools/stringstools.h index 2c5fb4bd4..80731c581 100644 --- a/cpp_src/tools/stringstools.h +++ b/cpp_src/tools/stringstools.h @@ -16,9 +16,9 @@ namespace reindexer { std::string escapeString(std::string_view str); std::string unescapeString(std::string_view str); -inline bool isalpha(char c) noexcept { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); } -inline bool isdigit(char c) noexcept { return (c >= '0' && c <= '9'); } -inline char tolower(char c) noexcept { return (c >= 'A' && c <= 'Z') ? c + 'a' - 'A' : c; } +[[nodiscard]] RX_ALWAYS_INLINE bool isalpha(char c) noexcept { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); } +[[nodiscard]] RX_ALWAYS_INLINE bool isdigit(char c) noexcept { return (c >= '0' && c <= '9'); } +[[nodiscard]] RX_ALWAYS_INLINE char tolower(char c) noexcept { return (c >= 'A' && c <= 'Z') ? c + 'a' - 'A' : c; } std::string toLower(std::string_view src); inline std::string_view skipSpace(std::string_view str) { size_t i = 0; @@ -28,7 +28,7 @@ inline std::string_view skipSpace(std::string_view str) { } template -Container& split(const typename Container::value_type& str, const std::string& delimiters, bool trimEmpty, Container& tokens) { +Container& split(const typename Container::value_type& str, std::string_view delimiters, bool trimEmpty, Container& tokens) { tokens.resize(0); for (size_t pos, lastPos = 0;; lastPos = pos + 1) { @@ -145,8 +145,22 @@ LogLevel logLevelFromString(const std::string& strLogLevel); StrictMode strictModeFromString(const std::string& strStrictMode); std::string_view strictModeToString(StrictMode mode); -bool iequals(std::string_view lhs, std::string_view rhs) noexcept; -bool iless(std::string_view lhs, std::string_view rhs) noexcept; +inline bool iequals(std::string_view lhs, std::string_view rhs) noexcept { + if (lhs.size() != rhs.size()) return false; + for (auto itl = lhs.begin(), itr = rhs.begin(); itl != lhs.end() && itr != rhs.end();) { + if (tolower(*itl++) != tolower(*itr++)) return false; + } + return true; +} +inline bool iless(std::string_view lhs, std::string_view rhs) noexcept { + const auto len = std::min(lhs.size(), rhs.size()); + for (size_t i = 0; i < len; ++i) { + if (const auto l = tolower(lhs[i]), r = tolower(rhs[i]); l != r) { + return l < r; + } + } + return lhs.size() < rhs.size(); +} enum class CaseSensitive : bool { No, Yes }; template diff --git a/cpp_src/tools/timetools.cc b/cpp_src/tools/timetools.cc index a2f98dd97..adfe3e395 100644 --- a/cpp_src/tools/timetools.cc +++ b/cpp_src/tools/timetools.cc @@ -15,21 +15,17 @@ using namespace std::string_view_literals; namespace reindexer { int64_t getTimeNow(std::string_view mode) { - auto tm = system_clock::now(); - auto duration = tm.time_since_epoch(); + const auto tm = system_clock::now(); + const auto duration = tm.time_since_epoch(); if (iequals(mode, "sec"sv)) { - auto cnt = duration_cast(duration).count(); - return static_cast(cnt); + return duration_cast(duration).count(); } else if (iequals(mode, "msec"sv)) { - auto cnt = duration_cast(duration).count(); - return static_cast(cnt); + return duration_cast(duration).count(); } else if (iequals(mode, "usec"sv)) { - auto cnt = duration_cast(duration).count(); - return static_cast(cnt); + return duration_cast(duration).count(); } else if (iequals(mode, "nsec"sv)) { - auto cnt = duration_cast(duration).count(); - return static_cast(cnt); + return duration_cast(duration).count(); } throw Error(errParams, "Unknown parameter '%s' in getTimeNow function.", mode); diff --git a/cpp_src/tools/timetools.h b/cpp_src/tools/timetools.h index 0ee6f9e1f..8a56634a6 100644 --- a/cpp_src/tools/timetools.h +++ b/cpp_src/tools/timetools.h @@ -1,5 +1,6 @@ #pragma once +#include #include namespace reindexer { diff --git a/cpp_src/vendor/sort/pdqsort.hpp b/cpp_src/vendor/sort/pdqsort.hpp index 980256977..f22bcdcd0 100644 --- a/cpp_src/vendor/sort/pdqsort.hpp +++ b/cpp_src/vendor/sort/pdqsort.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include diff --git a/cpp_src/vendor/sparse-map/sparse_hash.h b/cpp_src/vendor/sparse-map/sparse_hash.h index 909b4ab3f..5115de301 100644 --- a/cpp_src/vendor/sparse-map/sparse_hash.h +++ b/cpp_src/vendor/sparse-map/sparse_hash.h @@ -1809,7 +1809,8 @@ class sparse_hash : private Allocator, private Hash, private KeyEqual, private G template void insert_on_rehash(K &&key_value) { - const key_type &key = KeySelect()(key_value); + KeySelect ks; + const key_type &key = ks(key_value); const std::size_t hash = hash_key(key); std::size_t ibucket = bucket_for_hash(hash); diff --git a/iterator.go b/iterator.go index 4d8e84a0f..5d46e1777 100644 --- a/iterator.go +++ b/iterator.go @@ -16,6 +16,8 @@ import ( type ExplainSelector struct { // Field or index name Field string `json:"field"` + // Field type enum: indexed, non-indexed + FieldType string `json:"field_type,omitempty"` // Method, used to process condition Method string `json:"method"` // Number of uniq keys, processed by this selector (may be incorrect, in case of internal query optimization/caching @@ -39,6 +41,8 @@ type ExplainSelector struct { type ExplainResults struct { // Total query execution time TotalUs int `json:"total_us"` + // Query preselect build and select time + PreselectUs int `json:"preselect_us"` // Query prepare and optimize time PrepareUs int `json:"prepare_us"` // Indexes keys selection time @@ -55,6 +59,48 @@ type ExplainResults struct { SortByUncommittedIndex bool `json:"sort_by_uncommitted_index"` // Filter selectors, used to proccess query conditions Selectors []ExplainSelector `json:"selectors"` + // Explaining attempts to inject Join queries ON-conditions into the Main Query WHERE clause + OnConditionsInjections []ExplainJoinOnInjections `json:"on_conditions_injections,omitempty"` +} + +// Describes the process of a single JOIN-query ON-conditions injection into the Where clause of a main query +type ExplainJoinOnInjections struct { + // joinable ns name + RightNsName string `json:"namespace"` + // original ON-conditions clause. SQL-like string + JoinOnCondition string `json:"on_condition"` + // total amount of time spent on checking and substituting all conditions + TotalTimeUs int `json:"total_time_us"` + // result of injection attempt + Succeed bool `json:"success"` + // optional{succeed==false}. Explains condition injection failure + Reason string `json:"reason,omitempty"` + // by_value or select + Type string `json:"type"` + // Injected condition. SQL-like string + InjectedCondition string `json:"injected_condition"` + // individual conditions processing results + Conditions []ExplainConditionInjection `json:"conditions,omitempty"` +} + +// Describes an injection attempt of a single condition from the ON-clause of a JOIN-query +type ExplainConditionInjection struct { + // single condition from Join ON section. SQL-like string + InitialCondition string `json:"condition"` + // total time elapsed from injection attempt start till the end of substitution or rejection + TotalTime int `json:"total_time_us"` + // optoinal{JoinOnInjection.type == Select}. Explain raw string from Select subquery + Explain *ExplainResults `json:"explain_select,omitempty"` + // Optional. Aggregation type used in subquery + AggType string `json:"agg_type,omitempty"` + // result of injection attempt + Succeed bool `json:"success"` + // optional{succeed==false}. Explains condition injection failure + Reason string `json:"reason,omitempty"` + // substituted condition in QueryEntry. SQL-like string + NewCondition string `json:"new_condition"` + // resulting size of query values set + ValuesCount int `json:"values_count"` } func errIterator(err error) *Iterator { diff --git a/query.go b/query.go index 3af2ab58f..804fdbafe 100644 --- a/query.go +++ b/query.go @@ -435,7 +435,9 @@ func (q *Query) WhereString(index string, condition int, keys ...string) *Query return q } -// WhereUuid - Add where condition to DB query with UUID args +// WhereUuid - Add where condition to DB query with UUID args. +// This function applies binary encoding to the uuid value. +// 'index' MUST be declared as uuid index in this case func (q *Query) WhereUuid(index string, condition int, keys ...string) *Query { q.ser.PutVarCUInt(queryCondition).PutVString(index).PutVarCUInt(q.nextOp).PutVarCUInt(condition) diff --git a/readme.md b/readme.md index eecfd90c9..abcd4f415 100644 --- a/readme.md +++ b/readme.md @@ -940,8 +940,13 @@ type Item struct { ### Aggregations -Reindexer allows to retrieve aggregated results. Currently Average, Sum, Minimum, Maximum Facet and Distinct aggregations are supported. +Reindexer allows to retrieve aggregated results. Currently Count, CountCached, Average, Sum, Minimum, Maximum, Facet and +Distinct +aggregations are supported. +- `Count` - get total number of documents that meet the querie's conditions +- `CountCached` - get total number of documents that meet the querie's conditions. Result value will be cached and may + be reused by the other queries with CountCached aggregation - `AggregateMax` - get maximum field value - `AggregateMin` - get minimum field value - `AggregateSum` - get sum field value @@ -949,11 +954,19 @@ Reindexer allows to retrieve aggregated results. Currently Average, Sum, Minimum - `AggregateFacet` - get fields facet value - `Distinct` - get list of unique values of the field -In order to support aggregation, `Query` has methods `AggregateAvg`, `AggregateSum`, `AggregateMin`, `AggregateMax`, `AggregateFacet` and `Distinct` those should be called before the `Query` execution: this will ask reindexer to calculate data aggregations. -Aggregation Facet is applicable to multiple data columns and the result of that could be sorted by any data column or 'count' and cutted off by offset and limit. -In order to support this functionality method `AggregateFacet` returns `AggregationFacetRequest` which has methods `Sort`, `Limit` and `Offset`. +In order to support aggregation, `Query` has methods `AggregateAvg`, `AggregateSum`, `AggregateMin`, `AggregateMax` +, `AggregateFacet` and `Distinct` those should be called before the `Query` execution: this will ask reindexer to +calculate data aggregations. +Aggregation Facet is applicable to multiple data columns and the result of that could be sorted by any data column or ' +count' and cutted off by offset and limit. +In order to support this functionality method `AggregateFacet` returns `AggregationFacetRequest` which has +methods `Sort`, `Limit` and `Offset`. -To get aggregation results, `Iterator` has method `AggResults`: it is available after query execution and returns slice of results. +Queries with MERGE will apply aggregations from the main query to all the merged subqueries. Subqueries can not have +their own aggregations. Available aggregations for MERGE-queries are: Count, CountCached, Sum, Min and Max. + +To get aggregation results, `Iterator` has method `AggResults`: it is available after query execution and returns slice +of results. Example code for aggregate `items` by `price` and `name` diff --git a/test/composite_indexes_test.go b/test/composite_indexes_test.go index fd6bf7ba8..63182ac15 100644 --- a/test/composite_indexes_test.go +++ b/test/composite_indexes_test.go @@ -89,10 +89,11 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -112,10 +113,11 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -135,10 +137,11 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+third", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+third", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -158,10 +161,11 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -182,16 +186,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -212,16 +218,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -242,16 +250,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -273,16 +283,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -305,22 +317,25 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -343,22 +358,25 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -381,22 +399,25 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -421,22 +442,25 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -464,6 +488,7 @@ func TestCompositeIndexesSubstitution(t *testing.T) { }, { Field: "first2 or first1", + FieldType: "indexed", Method: "scan", Comparators: 2, Matched: 1, @@ -501,12 +526,14 @@ func TestCompositeIndexesSubstitution(t *testing.T) { Selectors: []expectedExplain{ { Field: "first2", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, }, { Field: "second2", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, @@ -518,12 +545,14 @@ func TestCompositeIndexesSubstitution(t *testing.T) { Selectors: []expectedExplain{ { Field: "second1", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 0, }, { Field: "first1", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 0, @@ -565,16 +594,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { Field: "(id and first1+first2)", Selectors: []expectedExplain{ { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, }, @@ -582,16 +613,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { Field: "or (second1+second2 and id)", Selectors: []expectedExplain{ { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 0, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 0, }, { - Field: "id", - Method: "index", - Keys: 1, - Matched: 0, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 0, }, }, }, @@ -623,18 +656,21 @@ func TestCompositeIndexesSubstitution(t *testing.T) { }, { Field: "first1", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, }, { Field: "first2 or second2", + FieldType: "indexed", Method: "scan", Comparators: 2, Matched: 1, }, { Field: "second1", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, @@ -660,13 +696,15 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "second2 or second1", + FieldType: "indexed", Method: "scan", Comparators: 2, Matched: 1, @@ -694,13 +732,15 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "second2 or second1", + FieldType: "indexed", Method: "scan", Comparators: 2, Matched: 1, @@ -731,16 +771,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -772,16 +814,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { - Field: "second1+second2", - Method: "index", - Keys: 1, - Matched: 1, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -813,16 +857,18 @@ func TestCompositeIndexesSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "second1+second2", - Method: "index", - Keys: 0, - Matched: 0, + Field: "second1+second2", + FieldType: "indexed", + Method: "index", + Keys: 0, + Matched: 0, }, { - Field: "first1+first2", - Method: "index", - Keys: 1, - Matched: 0, + Field: "first1+first2", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 0, }, }, "") }) @@ -854,10 +900,11 @@ func TestCompositeIndexesBestSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second+third", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second+third", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -883,10 +930,11 @@ func TestCompositeIndexesBestSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second+third+fourth", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second+third+fourth", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) @@ -910,13 +958,15 @@ func TestCompositeIndexesBestSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second+third", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second+third", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "fourth or id", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, @@ -943,13 +993,15 @@ func TestCompositeIndexesBestSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "fourth or third", + FieldType: "indexed", Method: "scan", Comparators: 2, Matched: 1, @@ -975,13 +1027,15 @@ func TestCompositeIndexesBestSubstitution(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "fourth or third", + FieldType: "indexed", Method: "scan", Comparators: 2, Matched: 1, @@ -1013,18 +1067,21 @@ func TestCompositeIndexesBestSubstitution(t *testing.T) { }, { Field: "fourth", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, }, { Field: "second or third", + FieldType: "indexed", Method: "scan", Comparators: 2, Matched: 1, }, { Field: "first", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, @@ -1082,13 +1139,15 @@ func TestCompositeSubstitutionLimit(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second+fourth", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second+fourth", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "third", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, @@ -1118,13 +1177,15 @@ func TestCompositeSubstitutionLimit(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second+third", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second+third", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "fourth", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, @@ -1154,19 +1215,22 @@ func TestCompositeSubstitutionLimit(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, { Field: "third", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, }, { Field: "fourth", + FieldType: "indexed", Method: "scan", Comparators: 1, Matched: 1, @@ -1194,10 +1258,11 @@ func TestCompositeSubstitutionLimit(t *testing.T) { printExplainRes(explainRes) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "first+second+third", - Method: "index", - Keys: 1, - Matched: 1, + Field: "first+second+third", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") }) diff --git a/test/eq_and_set_test.go b/test/eq_and_set_test.go index 2fa7e6008..b1a9c7203 100644 --- a/test/eq_and_set_test.go +++ b/test/eq_and_set_test.go @@ -153,11 +153,11 @@ func TestEmptySetAndEqWithoutIdx(t *testing.T) { require.NoError(t, err) t.Run("empty eq in where condition", func(t *testing.T) { - t.Skip("1288") query := DBD.Query(ns). WhereString("name", reindexer.EQ) - _, err := query.Exec().FetchAll() - require.Error(t, err, "For condition required at least 1 argument, but provided 0") + result, err := query.Exec().FetchAll() + require.NoError(t, err) + assert.Nil(t, result) }) t.Run("empty set in where condition", func(t *testing.T) { @@ -178,21 +178,21 @@ func TestEmptySetAndEqWithoutIdx(t *testing.T) { }) t.Run("set and empty eq in where condition", func(t *testing.T) { - t.Skip("1288") query := DBD.Query(ns). WhereString("name", reindexer.EQ). WhereInt("year", reindexer.SET, testItem.Year) - _, err := query.Exec().FetchAll() - require.Error(t, err, "For condition required at least 1 argument, but provided 0") + result, err := query.Exec().FetchAll() + require.NoError(t, err) + assert.Nil(t, result) }) t.Run("empty set and eq in where condition", func(t *testing.T) { - t.Skip("1288") query := DBD.Query(ns). WhereString("name", reindexer.EQ). WhereInt("year", reindexer.SET) - _, err := query.Exec().FetchAll() - require.Error(t, err, "For condition required at least 1 argument, but provided 0") + result, err := query.Exec().FetchAll() + require.NoError(t, err) + assert.Nil(t, result) }) } @@ -206,11 +206,12 @@ func TestEmptySetAndEqWithCompositeIdx(t *testing.T) { require.NoError(t, err) t.Run("empty eq in where condition", func(t *testing.T) { - t.Skip("1288") query := DBD.Query(ns). WhereString("subid", reindexer.EQ) - _, err := query.Exec().FetchAll() - require.Error(t, err, "For condition required at least 1 argument, but provided 0") + result, err := query.Exec().FetchAll() + require.NoError(t, err) + assert.Nil(t, result) + }) t.Run("empty set in where condition", func(t *testing.T) { @@ -231,21 +232,23 @@ func TestEmptySetAndEqWithCompositeIdx(t *testing.T) { }) t.Run("set and empty eq in where condition", func(t *testing.T) { - t.Skip("1288") query := DBD.Query(ns). WhereString("subid", reindexer.EQ). WhereInt32("id", reindexer.SET, testItem.ID) - _, err := query.Exec().FetchAll() - require.Error(t, err, "For condition required at least 1 argument, but provided 0") + result, err := query.Exec().FetchAll() + require.NoError(t, err) + assert.Nil(t, result) + }) t.Run("empty set and eq in where condition", func(t *testing.T) { - t.Skip("1288") query := DBD.Query(ns). WhereString("subid", reindexer.EQ). WhereInt("id", reindexer.SET) - _, err := query.Exec().FetchAll() - require.Error(t, err, "For condition required at least 1 argument, but provided 0") + result, err := query.Exec().FetchAll() + require.NoError(t, err) + assert.Nil(t, result) + }) } diff --git a/test/huge_items_test.go b/test/huge_items_test.go index 451a537fc..3ba2179c2 100644 --- a/test/huge_items_test.go +++ b/test/huge_items_test.go @@ -72,66 +72,75 @@ func FillTestItemWide(nsName string, typ reflect.Type, start int, count int) { func buildWideItemsExpectedExplain(initialIndexes int) []expectedExplain { resExplain := []expectedExplain{ { - Field: "id", - Method: "index", - Keys: 1, - Matched: 1, + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, } // Part of the expected explain set manually and may be changed. Result depends on the inrernal core's substitutions ordering resExplain = append(resExplain, expectedExplain{ Field: "field_6", + FieldType: "indexed", Method: "scan", Keys: 0, Matched: 1, Comparators: 1, }) resExplain = append(resExplain, expectedExplain{ - Field: "field_0+field_1+field_2+field_3", - Method: "index", - Keys: 1, - Matched: 1, + Field: "field_0+field_1+field_2+field_3", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }) resExplain = append(resExplain, expectedExplain{ - Field: "field_4", - Method: "index", - Keys: 1, - Matched: 1, + Field: "field_4", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }) resExplain = append(resExplain, expectedExplain{ - Field: "field_5", - Method: "index", - Keys: 1, - Matched: 1, + Field: "field_5", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }) resExplain = append(resExplain, expectedExplain{ - Field: "field_7", - Method: "index", - Keys: 1, - Matched: 1, + Field: "field_7", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }) resExplain = append(resExplain, expectedExplain{ - Field: "field_8", - Method: "index", - Keys: 1, - Matched: 1, + Field: "field_8", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }) for i := 9; i < initialIndexes-1; { var explainField expectedExplain if i%3 == 0 { explainField = expectedExplain{ - Field: fmt.Sprintf("field_%d+field_%d+field_%d+field_%d", i, i+1, i+2, i+3), - Method: "index", - Keys: 1, - Matched: 1, + Field: fmt.Sprintf("field_%d+field_%d+field_%d+field_%d", i, i+1, i+2, i+3), + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, } i += 4 } else { explainField = expectedExplain{ - Field: fmt.Sprintf("field_%d", i), - Method: "index", - Keys: 1, - Matched: 1, + Field: fmt.Sprintf("field_%d", i), + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, } i += 1 } diff --git a/test/join_test.go b/test/join_test.go index 0425231f8..6b6f42695 100644 --- a/test/join_test.go +++ b/test/join_test.go @@ -467,6 +467,7 @@ func initNsForExplain(t *testing.T, ns string, count int) { type expectedExplain struct { Field string + FieldType string Method string Keys int Comparators int @@ -476,6 +477,26 @@ type expectedExplain struct { Selectors []expectedExplain } +type expectedExplainConditionInjection struct { + InitialCondition string + AggType string + Succeed bool + Reason string + NewCondition string + ValuesCount int + ConditionSelectors []expectedExplain +} + +type expectedExplainJoinOnInjections struct { + RightNsName string + JoinOnCondition string + Succeed bool + Reason string + Type string + InjectedCondition string + Conditions []expectedExplainConditionInjection +} + func checkExplain(t *testing.T, res []reindexer.ExplainSelector, expected []expectedExplain, fieldName string) { require.Equal(t, len(expected), len(res)) for i := 0; i < len(expected); i++ { @@ -486,6 +507,7 @@ func checkExplain(t *testing.T, res []reindexer.ExplainSelector, expected []expe } else { assert.Equalf(t, len(res[i].Selectors), 0, fieldName+expected[i].Field) assert.Equalf(t, expected[i].Field, res[i].Field, fieldName+expected[i].Field) + assert.Equalf(t, expected[i].FieldType, res[i].FieldType, fieldName+expected[i].Field) assert.Equalf(t, expected[i].Method, res[i].Method, fieldName+expected[i].Field) assert.Equalf(t, expected[i].Matched, res[i].Matched, fieldName+expected[i].Field) assert.Equalf(t, expected[i].Keys, res[i].Keys, fieldName+expected[i].Field) @@ -504,6 +526,39 @@ func checkExplain(t *testing.T, res []reindexer.ExplainSelector, expected []expe } } +func checkExplainConditionInjection(t *testing.T, resConditions []reindexer.ExplainConditionInjection, expectedConditions []expectedExplainConditionInjection) { + for i := 0; i < len(expectedConditions); i++ { + assert.Equal(t, expectedConditions[i].InitialCondition, resConditions[i].InitialCondition) + assert.Equal(t, expectedConditions[i].AggType, resConditions[i].AggType) + assert.Equal(t, expectedConditions[i].Succeed, resConditions[i].Succeed) + assert.Equal(t, expectedConditions[i].Reason, resConditions[i].Reason) + assert.Equal(t, expectedConditions[i].NewCondition, resConditions[i].NewCondition) + assert.Equal(t, expectedConditions[i].ValuesCount, resConditions[i].ValuesCount) + if len(expectedConditions[i].ConditionSelectors) == 0 { + assert.Nil(t, resConditions[i].Explain) + } else { + checkExplain(t, resConditions[i].Explain.Selectors, expectedConditions[i].ConditionSelectors, "") + } + } +} + +func checkExplainJoinOnInjections(t *testing.T, res []reindexer.ExplainJoinOnInjections, expected []expectedExplainJoinOnInjections) { + require.Equal(t, len(expected), len(res)) + for i := 0; i < len(expected); i++ { + assert.Equal(t, expected[i].RightNsName, res[i].RightNsName) + assert.Equal(t, expected[i].JoinOnCondition, res[i].JoinOnCondition) + assert.Equal(t, expected[i].Succeed, res[i].Succeed) + assert.Equal(t, expected[i].Reason, res[i].Reason) + assert.Equal(t, expected[i].Type, res[i].Type) + assert.Equal(t, expected[i].InjectedCondition, res[i].InjectedCondition) + if len(expected[i].Conditions) == 0 { + assert.Nil(t, res[i].Conditions) + } else { + checkExplainConditionInjection(t, res[i].Conditions, expected[i].Conditions) + } + } +} + func TestExplainJoin(t *testing.T) { nsMain := "test_explain_main" nsJoined := "test_explain_joined" @@ -535,6 +590,7 @@ func TestExplainJoin(t *testing.T) { }, { Field: "not data", + FieldType: "indexed", Method: "index", Keys: 1, Comparators: 0, @@ -545,6 +601,7 @@ func TestExplainJoin(t *testing.T) { Selectors: []expectedExplain{ { Field: "id", + FieldType: "indexed", Method: "scan", Keys: 0, Comparators: 1, @@ -560,6 +617,7 @@ func TestExplainJoin(t *testing.T) { JoinSelect: []expectedExplain{ { Field: "id", + FieldType: "indexed", Method: "index", Keys: 1, Comparators: 0, @@ -567,6 +625,7 @@ func TestExplainJoin(t *testing.T) { }, { Field: "data", + FieldType: "indexed", Method: "scan", Keys: 0, Comparators: 1, @@ -581,6 +640,7 @@ func TestExplainJoin(t *testing.T) { Selectors: []expectedExplain{ { Field: "id", + FieldType: "indexed", Method: "scan", Keys: 0, Comparators: 1, @@ -595,6 +655,7 @@ func TestExplainJoin(t *testing.T) { Preselect: []expectedExplain{ { Field: "data", + FieldType: "indexed", Method: "index", Keys: 3, Comparators: 0, @@ -607,6 +668,7 @@ func TestExplainJoin(t *testing.T) { }, { Field: "or id", + FieldType: "indexed", Method: "index", Keys: 1, Comparators: 0, @@ -621,6 +683,7 @@ func TestExplainJoin(t *testing.T) { Preselect: []expectedExplain{ { Field: "data", + FieldType: "indexed", Method: "index", Keys: 1, Comparators: 0, @@ -630,4 +693,55 @@ func TestExplainJoin(t *testing.T) { JoinSelect: nil, }, }, "") + checkExplainJoinOnInjections(t, explainRes.OnConditionsInjections, []expectedExplainJoinOnInjections{ + { + RightNsName: "test_explain_joined", + JoinOnCondition: "INNER JOIN ON (test_explain_joined.id = id)", + Succeed: true, + Type: "select", + InjectedCondition: "(id IN (...) )", + Conditions: []expectedExplainConditionInjection{ + { + InitialCondition: "test_explain_joined.id = id", + AggType: "distinct", + Succeed: true, + NewCondition: "id IN (...)", + ValuesCount: 19, + ConditionSelectors: []expectedExplain{ + { + Field: "id", + FieldType: "indexed", + Method: "index", + Keys: 20, + Comparators: 0, + Matched: 20, + }, + { + Field: "data", + FieldType: "indexed", + Method: "scan", + Keys: 0, + Comparators: 1, + Matched: 19, + }, + }, + }, + }, + }, + { + RightNsName: "test_explain_joined", + JoinOnCondition: "OR INNER JOIN ON (test_explain_joined.id = id)", + Succeed: true, + Type: "by_value", + InjectedCondition: "(id IN (...) )", + Conditions: []expectedExplainConditionInjection{ + { + InitialCondition: "test_explain_joined.id = id", + Succeed: true, + NewCondition: "id IN (...)", + ValuesCount: 3, + }, + }, + }, + }) } diff --git a/test/queries_test.go b/test/queries_test.go index 8702e87b6..c89451d7f 100644 --- a/test/queries_test.go +++ b/test/queries_test.go @@ -67,6 +67,7 @@ type TestItem struct { Tmp string `reindex:"tmp,-"` Nested StrictTestNest `reindex:"-" json:"nested"` Uuid string `reindex:"uuid,hash,uuid" json:"uuid"` + UuidStore string `reindex:"uuid_store,-,uuid" json:"uuid_store"` UuidArray []string `reindex:"uuid_array,hash,uuid" json:"uuid_array"` _ struct{} `reindex:"id+tmp,,composite,pk"` _ struct{} `reindex:"age+genre,,composite"` @@ -101,7 +102,8 @@ type TestItemIDOnly struct { EndTime int `json:"end_time"` StartTime int `json:"start_time"` Tmp string `reindex:"tmp,-"` - Uuid string `reindex:"uuid,,uuid" json:"uuid"` + Uuid string `reindex:"uuid,hash,uuid" json:"uuid"` + UuidStore string `reindex:"uuid_store,-,uuid" json:"uuid_store"` UuidArray []string `reindex:"uuid_array,hash,uuid" json:"uuid_array"` _ struct{} `reindex:"id+tmp,,composite,pk"` } @@ -133,6 +135,7 @@ type TestItemWithSparse struct { StartTime int `reindex:"start_time,tree"` Tmp string `reindex:"tmp,-"` Uuid string `reindex:"uuid,hash,uuid" json:"uuid"` + UuidStore string `reindex:"uuid_store,-,uuid" json:"uuid_store"` UuidArray []string `reindex:"uuid_array,hash,uuid" json:"uuid_array"` _ struct{} `reindex:"id+tmp,,composite,pk"` _ struct{} `reindex:"age+genre,,composite"` @@ -287,6 +290,7 @@ func newTestItem(id int, pkgsCount int) interface{} { Name: randString(), }, Uuid: randUuid(), + UuidStore: randUuid(), UuidArray: randUuidArray(rand.Int() % 20), } } @@ -344,6 +348,7 @@ func newTestItemIDOnly(id int, pkgsCount int) interface{} { Name: randString(), }, Uuid: randUuid(), + UuidStore: randUuid(), UuidArray: randUuidArray(rand.Int() % 20), } } @@ -374,6 +379,7 @@ func newTestItemWithSparse(id int, pkgsCount int) interface{} { Name: randString(), }, Uuid: randUuid(), + UuidStore: randUuid(), UuidArray: randUuidArray(rand.Int() % 20), } } @@ -587,7 +593,7 @@ func TestSTDistanceWrappers(t *testing.T) { require.NoError(t, err) it2, err := DBD.Query(ns).DWithin(field1, searchPoint, distance).Sort(fmt.Sprintf("ST_Distance(%s, ST_GeomFromText('point(%f %f)'))", field1, sortPoint[0], sortPoint[1]), false).ExecToJson().FetchAll() require.NoError(t, err) - require.Equal(t, it1, it2) + require.Equal(t, string(it1), string(it2)) } }) t.Run("ST_Distance between fields", func(t *testing.T) { @@ -598,7 +604,7 @@ func TestSTDistanceWrappers(t *testing.T) { require.NoError(t, err) it2, err := DBD.Query(ns).DWithin(field1, searchPoint, distance).Sort(fmt.Sprintf("ST_Distance(%s, %s)", field1, field2), true).ExecToJson().FetchAll() require.NoError(t, err) - require.Equal(t, it1, it2) + require.Equal(t, string(it1), string(it2)) } }) } @@ -1084,14 +1090,30 @@ func callQueriesSequence(t *testing.T, namespace string, distinct []string, sort WhereUuid("uuid", reindexer.LT, randUuid()). ExecAndVerify(t) + newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).Debug(reindexer.TRACE). + WhereUuid("uuid", reindexer.SET, randUuidArray(rand.Int()%10)...). + ExecAndVerify(t) + newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).ReqTotal().Debug(reindexer.TRACE). - Not().Where("uuid_array", reindexer.SET, randUuidArray(rand.Int()%10)). + Not().Where("uuid_store", reindexer.EQ, randUuid()). + ExecAndVerify(t) + + newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).Debug(reindexer.TRACE). + WhereUuid("uuid_store", reindexer.LT, randUuid()). + ExecAndVerify(t) + + newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).ReqTotal().Debug(reindexer.TRACE). + WhereUuid("uuid_store", reindexer.SET, randUuidArray(rand.Int()%10)...). ExecAndVerify(t) newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).ReqTotal().Debug(reindexer.TRACE). WhereUuid("uuid_array", reindexer.SET, randUuidArray(rand.Int()%10)...). ExecAndVerify(t) + newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).ReqTotal().Debug(reindexer.TRACE). + Not().Where("uuid_array", reindexer.SET, randUuidArray(rand.Int()%10)). + ExecAndVerify(t) + if !testComposite { return } diff --git a/test/query_test.go b/test/query_test.go index 0bb2a738b..9b7daea79 100644 --- a/test/query_test.go +++ b/test/query_test.go @@ -240,6 +240,12 @@ func (tx *txTest) UpsertAsync(s interface{}, cmpl bindings.Completion) error { return tx.tx.UpsertAsync(s, cmpl) } +func (tx *txTest) DeleteAsync(s interface{}, cmpl bindings.Completion) error { + val := reflect.Indirect(reflect.ValueOf(s)) + tx.ns.items[getPK(tx.ns, val)] = s + return tx.tx.DeleteAsync(s, cmpl) +} + func (tx *txTest) Commit() (int, error) { res, err := tx.tx.CommitWithCount() tx.db.SetSyncRequired() @@ -409,6 +415,9 @@ func (qt *queryTest) Where(index string, condition int, keys interface{}) *query return qt.where(index, condition, keys) } +// WhereUUID - Add where condition with UUID args. +// This function applies binary encoding to the uuid value. +// 'index' MUST be declared as uuid index in this case func (qt *queryTest) WhereUuid(index string, condition int, keys ...string) *queryTest { qt.q.WhereUuid(index, condition, keys...) return qt.where(index, condition, keys) diff --git a/test/tx_test.go b/test/tx_test.go index 6477efaf1..719057668 100644 --- a/test/tx_test.go +++ b/test/tx_test.go @@ -213,3 +213,33 @@ func TestAsyncTxTimeout(t *testing.T) { // Check, that there are no changes in namespace CheckTYx(t, testTxAsyncTimeoutItemNs, count1) } + +func callTxMethod(t *testing.T, method func(interface{}, bindings.Completion) error) { + type OtherItem struct { + Id int64 + Data int64 + } + + err := method(OtherItem{ + Id: 0, + Data: 0, + }, func(err error) {}) + + assert.NotNil(t, err) + rerr, ok := err.(bindings.Error) + assert.True(t, ok) + assert.Equal(t, rerr.Code(), reindexer.ErrCodeParams) +} + +func TestRollbackAsyncOpWithIncorrectItem(t *testing.T) { + tx := newTestTx(DB, testTxItemNs) + + callTxMethod(t, tx.UpdateAsync) + callTxMethod(t, tx.UpsertAsync) + callTxMethod(t, tx.InsertAsync) + callTxMethod(t, tx.DeleteAsync) + + err := tx.Rollback() + + assert.Nil(t, err) +} diff --git a/test/uuid_test.go b/test/uuid_test.go index e381d4497..699cbab73 100644 --- a/test/uuid_test.go +++ b/test/uuid_test.go @@ -14,10 +14,12 @@ import ( ) type TestUuidStruct struct { - ID int `reindex:"id,,pk"` - Uuid string `reindex:"uuid,hash,uuid" json:"uuid"` - UuidArray []string `reindex:"uuid_array,hash,uuid" json:"uuid_array"` - _ struct{} `reindex:"id+uuid,,composite"` + ID int `reindex:"id,,pk"` + Uuid string `reindex:"uuid,hash,uuid" json:"uuid"` + UuidStore string `reindex:"uuid_store,-,uuid" json:"uuid_store"` + UuidArray []string `reindex:"uuid_array,hash,uuid" json:"uuid_array"` + UuidStoreArray []string `reindex:"uuid_store_array,-,uuid" json:"uuid_store_array"` + _ struct{} `reindex:"id+uuid,,composite"` } type TestIntItemWithUuidTagStruct struct { @@ -106,10 +108,11 @@ func checkExplainSelect(t *testing.T, it reindexer.Iterator, item interface{}) { assert.NoError(t, err) checkExplain(t, explainRes.Selectors, []expectedExplain{ { - Field: "uuid", - Method: "index", - Keys: 1, - Matched: 1, + Field: "uuid", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, }, }, "") } @@ -135,39 +138,98 @@ func TestUuidItemCreate(t *testing.T) { t.Run("add item with valid uuid", func(t *testing.T) { item := TestUuidStruct{ - ID: rand.Intn(100), Uuid: randUuid(), UuidArray: randUuidArray(rand.Intn(5)), + ID: rand.Intn(100), Uuid: randUuid(), UuidStore: randUuid(), + UuidArray: randUuidArray(rand.Intn(5)), UuidStoreArray: randUuidArray(rand.Intn(5)), } err := DB.Upsert(ns, item) require.NoError(t, err) }) - t.Run("add item with invalid uuid", func(t *testing.T) { - item := TestUuidStruct{ + t.Run("add item with invalid uuid: random string", func(t *testing.T) { + item1 := TestUuidStruct{ ID: rand.Intn(100), Uuid: randString(), UuidArray: randUuidArray(rand.Intn(5)), } + item2 := TestUuidStruct{ + ID: rand.Intn(100), UuidStore: randString(), UuidStoreArray: randUuidArray(rand.Intn(5)), + } + err1 := DB.Upsert(ns, item1) + require.Error(t, err1) + err2 := DB.Upsert(ns, item2) + require.Error(t, err2) + errExp := "UUID should consist of 32 hexadecimal digits" + assert.Contains(t, err1.Error(), errExp) + assert.Contains(t, err2.Error(), errExp) + }) + + t.Run("add item with invalid uuid: invalid char", func(t *testing.T) { + item := TestUuidStruct{ + ID: rand.Intn(100), Uuid: randUuid(), UuidArray: randUuidArray(rand.Intn(5)), + } + invalidUuid := []byte(item.Uuid) + invalidUuid[5] = 'x' + item.Uuid = string(invalidUuid) err := DB.Upsert(ns, item) require.Error(t, err) errExp := "UUID cannot contain char" assert.Contains(t, err.Error(), errExp) }) - t.Run("add item with invalid array uuid", func(t *testing.T) { + t.Run("add item with invalid uuid: invalid format", func(t *testing.T) { item := TestUuidStruct{ + ID: rand.Intn(100), Uuid: randUuid(), UuidArray: randUuidArray(rand.Intn(5)), + } + invalidUuid := []byte(item.Uuid) + invalidUuid[8] = invalidUuid[7] + invalidUuid[7] = '-' + item.Uuid = string(invalidUuid) + err := DB.Upsert(ns, item) + require.Error(t, err) + errExp := "Invalid UUID format" + assert.Contains(t, err.Error(), errExp) + }) + + t.Run("add item with invalid array uuid: random strings", func(t *testing.T) { + item1 := TestUuidStruct{ ID: rand.Intn(100), Uuid: randUuid(), UuidArray: randStringArr(rand.Intn(5) + 1), } + item2 := TestUuidStruct{ + ID: rand.Intn(100), UuidStore: randUuid(), UuidStoreArray: randStringArr(rand.Intn(5) + 1), + } + err1 := DB.Upsert(ns, item1) + require.Error(t, err1) + err2 := DB.Upsert(ns, item2) + require.Error(t, err2) + errExp := "UUID should consist of 32 hexadecimal digits" + assert.Contains(t, err1.Error(), errExp) + assert.Contains(t, err2.Error(), errExp) + }) + + t.Run("add item with invalid array uuid: invalid char", func(t *testing.T) { + item := TestUuidStruct{ + ID: rand.Intn(100), Uuid: randUuid(), UuidArray: randUuidArray(rand.Intn(5) + 1), + } + invalidUuid := []byte(item.UuidArray[0]) + invalidUuid[15] = '%' + item.UuidArray[0] = string(invalidUuid) err := DB.Upsert(ns, item) require.Error(t, err) errExp := "UUID cannot contain char" assert.Contains(t, err.Error(), errExp) }) - t.Run("add item with invalid uuid and array uuid", func(t *testing.T) { + t.Run("add item with invalid uuid and array uuid: invalid format", func(t *testing.T) { item := TestUuidStruct{ - ID: rand.Intn(100), Uuid: randString(), UuidArray: randStringArr(rand.Intn(5) + 1), + ID: rand.Intn(100), Uuid: randUuid(), UuidArray: randUuidArray(rand.Intn(5) + 1), } + invalidUuid := []byte(item.Uuid) + invalidUuid[0] = '-' + item.Uuid = string(invalidUuid) + invalidUuid = []byte(item.UuidArray[0]) + invalidUuid[30] = '-' + item.UuidArray[0] = string(invalidUuid) err := DB.Upsert(ns, item) require.Error(t, err) - errExp := "UUID cannot contain char" + errExp := "Invalid UUID format" assert.Contains(t, err.Error(), errExp) }) } diff --git a/tx.go b/tx.go index 31e41ea78..8fb6be4c5 100644 --- a/tx.go +++ b/tx.go @@ -191,8 +191,21 @@ func (tx *Tx) DeleteJSON(json []byte, precepts ...string) error { return tx.modifyInternal(nil, json, modeDelete, precepts...) } +func (tx *Tx) handleRecoverFromAsyncOp(rec interface{}) error { + tx.cmplCond.L.Lock() + atomic.AddUint32(&tx.asyncRspCnt, ^uint32(0)) + tx.cmplCond.Broadcast() + tx.cmplCond.L.Unlock() + switch x := rec.(type) { + case error: + return x + default: + return bindings.NewError("Unknown panic type", ErrCodeLogic) + } +} + // InsertAsync Insert item to namespace. Calls completion on result -func (tx *Tx) InsertAsync(item interface{}, cmpl bindings.Completion, precepts ...string) error { +func (tx *Tx) InsertAsync(item interface{}, cmpl bindings.Completion, precepts ...string) (err error) { if tx.db.otelTracer != nil { defer tx.db.startTracingSpan(tx.ctx.UserCtx, "Reindexer.Tx.InsertAsync", otelattr.String("rx.ns", tx.namespace)).End() } @@ -201,7 +214,7 @@ func (tx *Tx) InsertAsync(item interface{}, cmpl bindings.Completion, precepts . defer prometheus.NewTimer(tx.db.promMetrics.clientCallsLatency.WithLabelValues("Tx.InsertAsync", tx.namespace)).ObserveDuration() } - err := tx.startTx() + err = tx.startTx() if err != nil { return err } @@ -210,11 +223,17 @@ func (tx *Tx) InsertAsync(item interface{}, cmpl bindings.Completion, precepts . return err } + defer func() { + if rec := recover(); rec != nil { + err = tx.handleRecoverFromAsyncOp(rec) + } + }() + return tx.modifyInternalAsync(item, nil, modeInsert, cmpl, retriesOnInvalidStateCnt, precepts...) } // UpdateAsync Update item to namespace. Calls completion on result -func (tx *Tx) UpdateAsync(item interface{}, cmpl bindings.Completion, precepts ...string) error { +func (tx *Tx) UpdateAsync(item interface{}, cmpl bindings.Completion, precepts ...string) (err error) { if tx.db.otelTracer != nil { defer tx.db.startTracingSpan(tx.ctx.UserCtx, "Reindexer.Tx.UpdateAsync", otelattr.String("rx.ns", tx.namespace)).End() } @@ -223,7 +242,7 @@ func (tx *Tx) UpdateAsync(item interface{}, cmpl bindings.Completion, precepts . defer prometheus.NewTimer(tx.db.promMetrics.clientCallsLatency.WithLabelValues("Tx.UpdateAsync", tx.namespace)).ObserveDuration() } - err := tx.startTx() + err = tx.startTx() if err != nil { return err } @@ -232,11 +251,17 @@ func (tx *Tx) UpdateAsync(item interface{}, cmpl bindings.Completion, precepts . return err } + defer func() { + if rec := recover(); rec != nil { + err = tx.handleRecoverFromAsyncOp(rec) + } + }() + return tx.modifyInternalAsync(item, nil, modeUpdate, cmpl, retriesOnInvalidStateCnt, precepts...) } // UpsertAsync (Insert or Update) item to namespace. Calls completion on result -func (tx *Tx) UpsertAsync(item interface{}, cmpl bindings.Completion, precepts ...string) error { +func (tx *Tx) UpsertAsync(item interface{}, cmpl bindings.Completion, precepts ...string) (err error) { if tx.db.otelTracer != nil { defer tx.db.startTracingSpan(tx.ctx.UserCtx, "Reindexer.Tx.UpsertAsync", otelattr.String("rx.ns", tx.namespace)).End() } @@ -245,7 +270,7 @@ func (tx *Tx) UpsertAsync(item interface{}, cmpl bindings.Completion, precepts . defer prometheus.NewTimer(tx.db.promMetrics.clientCallsLatency.WithLabelValues("Tx.UpsertAsync", tx.namespace)).ObserveDuration() } - err := tx.startTx() + err = tx.startTx() if err != nil { return err } @@ -254,6 +279,12 @@ func (tx *Tx) UpsertAsync(item interface{}, cmpl bindings.Completion, precepts . return err } + defer func() { + if rec := recover(); rec != nil { + err = tx.handleRecoverFromAsyncOp(rec) + } + }() + return tx.modifyInternalAsync(item, nil, modeUpsert, cmpl, retriesOnInvalidStateCnt, precepts...) } @@ -280,7 +311,7 @@ func (tx *Tx) UpsertJSONAsync(json []byte, cmpl bindings.Completion, precepts .. } // DeleteAsync - remove item by id from namespace. Calls completion on result -func (tx *Tx) DeleteAsync(item interface{}, cmpl bindings.Completion, precepts ...string) error { +func (tx *Tx) DeleteAsync(item interface{}, cmpl bindings.Completion, precepts ...string) (err error) { if tx.db.otelTracer != nil { defer tx.db.startTracingSpan(tx.ctx.UserCtx, "Reindexer.Tx.DeleteAsync", otelattr.String("rx.ns", tx.namespace)).End() } @@ -289,7 +320,7 @@ func (tx *Tx) DeleteAsync(item interface{}, cmpl bindings.Completion, precepts . defer prometheus.NewTimer(tx.db.promMetrics.clientCallsLatency.WithLabelValues("Tx.DeleteAsync", tx.namespace)).ObserveDuration() } - err := tx.startTx() + err = tx.startTx() if err != nil { return err } @@ -298,6 +329,12 @@ func (tx *Tx) DeleteAsync(item interface{}, cmpl bindings.Completion, precepts . return err } + defer func() { + if rec := recover(); rec != nil { + err = tx.handleRecoverFromAsyncOp(rec) + } + }() + return tx.modifyInternalAsync(item, nil, modeDelete, cmpl, retriesOnInvalidStateCnt, precepts...) }