diff --git a/bindings.go b/bindings.go index abbf382d3..dffa2def8 100644 --- a/bindings.go +++ b/bindings.go @@ -58,7 +58,7 @@ func (db *reindexerImpl) modifyItem(ctx context.Context, namespace string, ns *r rdSer := newSerializer(out.GetBuf()) rawQueryParams := rdSer.readRawQueryParams(func(nsid int) { - ns.cjsonState.ReadPayloadType(&rdSer.Serializer) + ns.cjsonState.ReadPayloadType(&rdSer.Serializer, db.binding, ns.name) }) if rawQueryParams.count == 0 { @@ -71,7 +71,7 @@ func (db *reindexerImpl) modifyItem(ctx context.Context, namespace string, ns *r if len(precepts) > 0 && (resultp.cptr != 0 || resultp.data != nil) && reflect.TypeOf(item).Kind() == reflect.Ptr { nsArrEntry := nsArrayEntry{ns, ns.cjsonState.Copy()} - if _, err := unpackItem(&nsArrEntry, &resultp, false, true, item); err != nil { + if _, err := unpackItem(db.binding, &nsArrEntry, &resultp, false, true, item); err != nil { return 0, err } } @@ -123,7 +123,7 @@ func (db *reindexerImpl) getNS(namespace string) (*reindexerNamespace, error) { return ns, nil } -func unpackItem(ns *nsArrayEntry, params *rawResultItemParams, allowUnsafe bool, nonCacheableData bool, item interface{}) (interface{}, error) { +func unpackItem(bin bindings.RawBinding, ns *nsArrayEntry, params *rawResultItemParams, allowUnsafe bool, nonCacheableData bool, item interface{}) (interface{}, error) { useCache := item == nil && (ns.deepCopyIface || allowUnsafe) && !nonCacheableData needCopy := ns.deepCopyIface && !allowUnsafe var err error @@ -133,7 +133,7 @@ func unpackItem(ns *nsArrayEntry, params *rawResultItemParams, allowUnsafe bool, item = citem.item } else { item = reflect.New(ns.rtype).Interface() - dec := ns.localCjsonState.NewDecoder(item, logger) + dec := ns.localCjsonState.NewDecoder(item, bin) if params.cptr != 0 { err = dec.DecodeCPtr(params.cptr, item) } else if params.data != nil { @@ -159,7 +159,7 @@ func unpackItem(ns *nsArrayEntry, params *rawResultItemParams, allowUnsafe bool, if item == nil { item = reflect.New(ns.rtype).Interface() } - dec := ns.localCjsonState.NewDecoder(item, logger) + dec := ns.localCjsonState.NewDecoder(item, bin) if params.cptr != 0 { err = dec.DecodeCPtr(params.cptr, item) } else if params.data != nil { @@ -425,7 +425,7 @@ func (db *reindexerImpl) updateQuery(ctx context.Context, q *Query) *Iterator { ser := newSerializer(result.GetBuf()) // skip total count rawQueryParams := ser.readRawQueryParams(func(nsid int) { - ns.cjsonState.ReadPayloadType(&ser.Serializer) + ns.cjsonState.ReadPayloadType(&ser.Serializer, db.binding, ns.name) }) for i := 0; i < rawQueryParams.count; i++ { diff --git a/bindings/builtin/builtin.go b/bindings/builtin/builtin.go index bfd8b17ed..a527bc1c6 100644 --- a/bindings/builtin/builtin.go +++ b/bindings/builtin/builtin.go @@ -31,8 +31,15 @@ type Logger interface { Printf(level int, fmt string, msg ...interface{}) } -var logger Logger +// Separate mutexes for logger object itself and for reindexer_enable_logger call: +// logMtx provides safe access to the logger +// logEnableMtx provides atomic logic for (enable + set) and (disable + reset) procedures +// This logger is global to easily export it into CGO (however it may lead to some confusion if there are multiple builtin instances in the app) var logMtx sync.RWMutex +var logEnableMtx sync.Mutex +var logger Logger +var emptyLogger bindings.NullLogger + var enableDebug bool var bufPool sync.Pool @@ -600,18 +607,33 @@ func CGoLogger(level int, msg string) { } } -func (binding *Builtin) EnableLogger(log bindings.Logger) { +func (binding *Builtin) setLogger(log bindings.Logger) { logMtx.Lock() defer logMtx.Unlock() logger = log +} + +func (binding *Builtin) EnableLogger(log bindings.Logger) { + logEnableMtx.Lock() + defer logEnableMtx.Unlock() + binding.setLogger(log) C.reindexer_enable_go_logger() } func (binding *Builtin) DisableLogger() { - logMtx.Lock() - defer logMtx.Unlock() + logEnableMtx.Lock() + defer logEnableMtx.Unlock() C.reindexer_disable_go_logger() - logger = nil + binding.setLogger(nil) +} + +func (binding *Builtin) GetLogger() bindings.Logger { + logMtx.RLock() + defer logMtx.RUnlock() + if logger != nil { + return logger + } + return &emptyLogger } func (binding *Builtin) ReopenLogFiles() error { diff --git a/bindings/builtinserver/builtinserver.go b/bindings/builtinserver/builtinserver.go index 69a9ade65..e568968fa 100644 --- a/bindings/builtinserver/builtinserver.go +++ b/bindings/builtinserver/builtinserver.go @@ -249,6 +249,10 @@ func (server *BuiltinServer) DisableLogger() { server.builtin.DisableLogger() } +func (server *BuiltinServer) GetLogger() bindings.Logger { + return server.builtin.GetLogger() +} + func (server *BuiltinServer) ReopenLogFiles() error { return err2go(C.reopen_log_files(server.svc)) } diff --git a/bindings/builtinserver/config/config.go b/bindings/builtinserver/config/config.go index 30ffe0516..4ae1d7b99 100644 --- a/bindings/builtinserver/config/config.go +++ b/bindings/builtinserver/config/config.go @@ -13,11 +13,20 @@ type StorageConf struct { Autorepair bool `yaml:"autorepair"` } +const ServerThreadingDedicated = "dedicated" +const ServerThreadingShared = "shared" + type NetConf struct { - HTTPAddr string `yaml:"httpaddr"` - RPCAddr string `yaml:"rpcaddr"` - WebRoot string `yaml:"webroot"` - Security bool `yaml:"security"` + HTTPAddr string `yaml:"httpaddr"` + HTTPThreading string `yaml:"http_threading"` // "dedicated" or "shared" + RPCAddr string `yaml:"rpcaddr"` + RPCThreading string `yaml:"rpc_threading"` // "dedicated" or "shared" + UnixRPCAddr string `yaml:"urpcaddr"` + UnixRPCThreading string `yaml:"urpc_threading"` // "dedicated" or "shared" + WebRoot string `yaml:"webroot"` + Security bool `yaml:"security"` + HttpReadTimeoutSec int `yaml:"http_read_timeout,omitempty"` + HttpWriteTimeoutSec int `yaml:"http_write_timeout,omitempty"` } type LoggerConf struct { @@ -69,9 +78,11 @@ func DefaultServerConfig() *ServerConfig { Autorepair: false, }, Net: NetConf{ - HTTPAddr: "0.0.0.0:9088", - RPCAddr: "0.0.0.0:6534", - Security: false, + HTTPAddr: "0.0.0.0:9088", + HTTPThreading: "shared", + RPCAddr: "0.0.0.0:6534", + RPCThreading: "shared", + Security: false, }, Logger: LoggerConf{ ServerLog: "stdout", diff --git a/bindings/consts.go b/bindings/consts.go index df37a81f7..504f0a3e6 100644 --- a/bindings/consts.go +++ b/bindings/consts.go @@ -2,7 +2,7 @@ package bindings const CInt32Max = int(^uint32(0) >> 1) -const ReindexerVersion = "v3.19.0" +const ReindexerVersion = "v3.20.0" // public go consts from type_consts.h and reindexer_ctypes.h const ( diff --git a/bindings/cproto/connection.go b/bindings/cproto/connection.go index 3a9d165d8..136b316e9 100644 --- a/bindings/cproto/connection.go +++ b/bindings/cproto/connection.go @@ -198,12 +198,20 @@ func (c *connection) deadlineTicker() { } func (c *connection) connect(ctx context.Context) (err error) { + dsn := c.owner.getActiveDSN() var d net.Dialer - c.conn, err = d.DialContext(ctx, "tcp", c.owner.getActiveDSN().Host) - if err != nil { - return err + if dsn.Scheme == "cproto" { + if c.conn, err = d.DialContext(ctx, "tcp", dsn.Host); err != nil { + return err + } + c.conn.(*net.TCPConn).SetNoDelay(true) + } else { + d.LocalAddr = nil + if c.conn, err = d.DialContext(ctx, "unix", dsn.Host); err != nil { + return err + } } - c.conn.(*net.TCPConn).SetNoDelay(true) + c.rdBuf = bufio.NewReaderSize(c.conn, bufsCap) go c.writeLoop() diff --git a/bindings/cproto/cproto.go b/bindings/cproto/cproto.go index bc6ab855b..c5ceb476a 100644 --- a/bindings/cproto/cproto.go +++ b/bindings/cproto/cproto.go @@ -8,6 +8,8 @@ import ( "math" "net" "net/url" + "runtime" + "strings" "sync" "sync/atomic" "time" @@ -27,11 +29,13 @@ const ( opWr = 1 ) -var logger Logger -var logMtx sync.RWMutex +var emptyLogger bindings.NullLogger func init() { bindings.RegisterBinding("cproto", new(NetCProto)) + if runtime.GOOS != "windows" { + bindings.RegisterBinding("ucproto", new(NetCProto)) + } } type Logger interface { @@ -52,6 +56,8 @@ type NetCProto struct { appName string termCh chan struct{} lock sync.RWMutex + logger Logger + logMtx sync.RWMutex } type dsn struct { @@ -122,6 +128,16 @@ func (binding *NetCProto) Init(u []url.URL, options ...interface{}) (err error) } binding.dsn.url = u + for i := 0; i < len(binding.dsn.url); i++ { + if binding.dsn.url[i].Scheme == "ucproto" { + addrs := strings.Split(binding.dsn.url[i].Path, ":") + if len(addrs) != 2 { + return fmt.Errorf("rq: unexpected URL format for ucproto: '%s'. Expecting ':/", binding.dsn.url[i].Path) + } + binding.dsn.url[i].Host = addrs[0] + binding.dsn.url[i].Path = addrs[1] + } + } binding.connectDSN(context.Background(), connPoolSize, connPoolLBAlgorithm) binding.termCh = make(chan struct{}) go binding.pinger() @@ -391,16 +407,26 @@ func (binding *NetCProto) OnChangeCallback(f func()) { } func (binding *NetCProto) EnableLogger(log bindings.Logger) { - logMtx.Lock() - defer logMtx.Unlock() - logger = log + binding.logMtx.Lock() + defer binding.logMtx.Unlock() + binding.logger = log } func (binding *NetCProto) DisableLogger() { - logMtx.Lock() - defer logMtx.Unlock() - logger = nil + binding.logMtx.Lock() + defer binding.logMtx.Unlock() + binding.logger = nil } + +func (binding *NetCProto) GetLogger() bindings.Logger { + binding.logMtx.RLock() + defer binding.logMtx.RUnlock() + if binding.logger != nil { + return binding.logger + } + return &emptyLogger +} + func (binding *NetCProto) ReopenLogFiles() error { fmt.Println("cproto binding ReopenLogFiles method is dummy") return nil @@ -465,10 +491,10 @@ func (binding *NetCProto) getAllConns() []*connection { } func (binding *NetCProto) logMsg(level int, fmt string, msg ...interface{}) { - logMtx.RLock() - defer logMtx.RUnlock() - if logger != nil { - logger.Printf(level, fmt, msg) + binding.logMtx.RLock() + defer binding.logMtx.RUnlock() + if binding.logger != nil { + binding.logger.Printf(level, fmt, msg) } } diff --git a/bindings/interface.go b/bindings/interface.go index 221296b78..1f9e28d99 100644 --- a/bindings/interface.go +++ b/bindings/interface.go @@ -120,7 +120,7 @@ type TxCtx struct { UserCtx context.Context } -// FetchMore interface for partial loading results (used in cproto) +// FetchMore interface for partial loading results (used in cproto/ucproto) type FetchMore interface { Fetch(ctx context.Context, offset, limit int, asJson bool) (err error) } @@ -130,6 +130,12 @@ type Logger interface { Printf(level int, fmt string, msg ...interface{}) } +type NullLogger struct { +} + +func (NullLogger) Printf(level int, fmt string, msg ...interface{}) { +} + func NewError(text string, code int) error { return Error{text, code} } @@ -196,6 +202,7 @@ type RawBinding interface { Commit(ctx context.Context, namespace string) error EnableLogger(logger Logger) DisableLogger() + GetLogger() Logger ReopenLogFiles() error Ping(ctx context.Context) error Finalize() error @@ -248,7 +255,7 @@ const ( LBPowerOfTwoChoices ) -// OptionConnPoolLoadBalancing sets algorithm, which will be used to choose connection for cproto requests' balancing +// OptionConnPoolLoadBalancing sets algorithm, which will be used to choose connection for cproto/ucproto requests' balancing type OptionConnPoolLoadBalancing struct { Algorithm LoadBalancingAlgorithm } diff --git a/changelog.md b/changelog.md index 9c3b0da69..eb97dd501 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,56 @@ +# Version 3.20.0 (30.10.2023) +## Core +- [fea] Added crash query report for Update/Delete queries +- [fea] Added configs for LRU index cache size (check section `cache` in the `namespaces` entity of the `#config`-namespace) +- [fea] Optimized CJSON deserialization +- [fea] Optimized WAL size (unchaged indexes and schemas will not be added into the WAL anymore) +- [fea] Added atomic rollback for multiple field UPDATE-queries in case of errors during query execution (currently it is atomic on the level of the individual document) +- [fix] Fixed column indexes optimization (some of the comparators have become noticeably more effective) +- [fix] Added PK check for UPDATE-queries +- [fix] Fixed JOINs on composite indexes +- [fix] Fixed select functions with MERGE queries for cases when the same namespace is merged multiple times +- [fix] Fixed non-indexed string/int fields convertion on index creation in cases when index type is not equal to the field type +- [fix] Disabled leveldb's log files (in rare cases those logs could lead to the problems with storage reopenning) +- [fix] Disable background indexes optimization for the temporary namespaces +- [fix] Removed attempts to reopen storage with flush errors befors it's destruction +- [fix] Some of the storage flushes were moved outside of the unique namespaces lock +- [fix] Fixed directories deletion for Windows + +## Replication +- [fea] Reduced memory consumptio for online-updates +- [fix] Fixed updates size calculation during max allowed updates size check (now it checks actual allocated memory, not the data size) +- [fix] Namespaces config applying after FORCE-sync replication +- [fix] Fixed some rare tagsmatcher conflicts in replicated namespaces in case when namespace was not replicted previously and had some data in it +- [fix] Fixed some unnecessary force syncs after go clients connection +- [fix] Fixed documents duplication after PK change in UPDATE-query +- [fix] Fixed logical race in cascade replication resync + +## Reindexer server +- [fea] Added support for UNIX domain sockets for RPC-server. Disabled by default (check `urpcaddr`-flag in server's config) +- [fea] Added support for default timeouts in HTTP-server (support for `http-write-timeout` and `http-read-timeout` options in server's config) +- [fea] Added support for `Request-Timeout` header in HTTP-requests to setup individual timeout for each request + +## Reindexer tool +- [fea] Added support for UNIX domain sockets (dsn format: `ucproto://:/`, example: `ucproto:///tmp/reindexer.sock:/my_db`) + +## Go connector +- [fea] Added support for UNIX domain sockets in cproto-binding (dsn format: `ucproto://:/`, example: `ucproto:///tmp/reindexer.sock:/my_db`) +- [fix] Fixed deadlock in `EnableLogger` method betweenn Go and C++ mutexes + +## Face +- [fea] Added the ability to use hot keys to navigate over the UI +- [fea] Added the link to the documentation to the left bar menu +- [fea] Changed the column filter with the case sensitive one +- [fea] Added validation of the JSON paths field to the Index Config page +- [fea] Added the wal_size field to the Namespace config +- [fea] Added the preselect_us and field_type sections fto the Explain page +- [fix] Fixed the horizontal scroll for the Grid view +- [fix] Fixed the data sorting on the Items page during the namespace changing +- [fix] Fixed the "undefined items" error during the db changing +- [fix] Fixed the console issues for the ttl index +- [fix] Made the Namespace refactoring +- [fix] Fixed the operations with the last column for the Grid view + # Version 3.19.0 (16.09.2023) ## Core - [fea] Added background namespaces deleteion. Previously in some cases atomic transactions could cause the namespace's deletion in the querie's execution thread, which lead to the latency spike @@ -1905,5 +1958,3 @@ - [ref] EnableStorage method was deprecated - [fix] Query builder did not reset opOR after InnerJoin -## Misc - diff --git a/cjson/decoder.go b/cjson/decoder.go index 3cc9bd183..d830c5fc7 100644 --- a/cjson/decoder.go +++ b/cjson/decoder.go @@ -8,6 +8,8 @@ import ( "strconv" "time" "unsafe" + + "github.com/restream/reindexer/v3/bindings" ) var ( @@ -15,15 +17,15 @@ var ( ifaceSliceType = reflect.TypeOf(ifaceSlice) ) -type Logger interface { - Printf(level int, fmt string, msg ...interface{}) +type LoggerOwner interface { + GetLogger() bindings.Logger } type Decoder struct { - ser *Serializer - state *State - ctagsCache *ctagsCache - logger Logger + ser *Serializer + state *State + ctagsCache *ctagsCache + loggerOwner LoggerOwner } const MaxIndexes = 256 @@ -658,19 +660,23 @@ func (dec *Decoder) DecodeCPtr(cptr uintptr, dest interface{}) (err error) { defer func() { if ret := recover(); ret != nil { - if dec.logger != nil { - dec.logger.Printf(1, - "Interface: %#v\nRead position: %d\nTags(v%d): %v\nPayload Type: %+v\nPayload Value: %v\nTags cache: %+v\nData dump:\n%s\nError: %v\n", - dest, - ser.Pos(), - dec.state.Version, - dec.state.tagsMatcher.Names, - dec.state.payloadType.Fields, - pl.getAsMap(), - dec.state.structCache, - hex.Dump(ser.Bytes()), - ret, - ) + if dec.loggerOwner != nil { + if logger := dec.loggerOwner.GetLogger(); logger != nil { + logger.Printf(1, + "Interface: %#v\nRead position: %d\nTags(v%d): %v\nPayload Type: %+v\nPayload Value: %v\nTags cache: %+v\nData dump:\n%s\nError: %v\nTagsMatcher: { state_token: 0x%08X, version: %d }\n", + dest, + ser.Pos(), + dec.state.Version, + dec.state.tagsMatcher.Names, + dec.state.payloadType.Fields, + pl.getAsMap(), + dec.state.structCache, + hex.Dump(ser.Bytes()), + ret, + uint32(dec.state.StateToken), + dec.state.Version, + ) + } } err = ret.(error) } @@ -695,23 +701,27 @@ func (dec *Decoder) Decode(cjson []byte, dest interface{}) (err error) { defer func() { if ret := recover(); ret != nil { - if dec.logger != nil { - dec.logger.Printf(1, - "Interface: %#v\nRead position: %d\nTags(v%d): %v\nTags cache: %+v\nData dump:\n%s\nError: %v\n", - dest, - ser.Pos(), - dec.state.Version, - dec.state.tagsMatcher.Names, - dec.state.structCache, - hex.Dump(ser.Bytes()), - ret, - ) + if dec.loggerOwner != nil { + if logger := dec.loggerOwner.GetLogger(); logger != nil { + logger.Printf(1, + "Interface: %#v\nRead position: %d\nTags(v%d): %v\nTags cache: %+v\nData dump:\n%s\nError: %v\nTagsMatcher: { state_token: 0x%08X, version: %d }\n", + dest, + ser.Pos(), + dec.state.Version, + dec.state.tagsMatcher.Names, + dec.state.structCache, + hex.Dump(ser.Bytes()), + ret, + uint32(dec.state.StateToken), + dec.state.Version, + ) + } } err = ret.(error) } }() - fieldsoutcnt := make([]int, 64, 64) + fieldsoutcnt := make([]int, MaxIndexes) ctagsPath := make([]int, 0, 8) dec.decodeValue(nil, ser, reflect.ValueOf(dest), fieldsoutcnt, ctagsPath) diff --git a/cjson/state.go b/cjson/state.go index ef6080144..f43f7d2d4 100644 --- a/cjson/state.go +++ b/cjson/state.go @@ -36,7 +36,7 @@ func (state *State) Copy() State { } } -func (state *State) ReadPayloadType(s *Serializer) State { +func (state *State) ReadPayloadType(s *Serializer, loggerOwner LoggerOwner, ns string) State { state.lock.Lock() defer state.lock.Unlock() stateToken := int32(s.GetVarUInt()) @@ -44,6 +44,12 @@ func (state *State) ReadPayloadType(s *Serializer) State { skip := state.Version >= version && state.StateToken == stateToken if !skip { + if loggerOwner != nil { + if logger := loggerOwner.GetLogger(); logger != nil { + logger.Printf(3, "rq: '%s' TagsMatcher was updated: { state_token: 0x%08X, version: %d } -> { state_token: 0x%08X, version: %d }", + ns, state.StateToken, state.Version, uint32(stateToken), version) + } + } state.StateData = &StateData{Version: version, StateToken: stateToken} } state.tagsMatcher.Read(s, skip) @@ -57,10 +63,10 @@ func (state *State) NewEncoder() Encoder { } } -func (state *State) NewDecoder(item interface{}, logger Logger) Decoder { +func (state *State) NewDecoder(item interface{}, loggerOwner LoggerOwner) Decoder { dec := Decoder{ - state: state, - logger: logger, + state: state, + loggerOwner: loggerOwner, } dec.state.sCacheLock.RLock() diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index df73c53cd..8bfc3150d 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -36,7 +36,7 @@ else() option (LINK_RESOURCES "Link web resources as binary data" ON) endif() -set (REINDEXER_VERSION_DEFAULT "3.19.0") +set (REINDEXER_VERSION_DEFAULT "3.20.0") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") @@ -89,6 +89,10 @@ endif () set (EXTRA_FLAGS "") +if (WITH_ASAN AND WITH_TSAN) + message(FATAL_ERROR "You cannot use the ASAN and TSAN options at the same time, CMake will exit.") +endif() + if (WITH_ASAN) set (EXTRA_FLAGS "-fsanitize=address") add_definitions(-DREINDEX_WITH_ASAN) @@ -272,6 +276,9 @@ else() list(APPEND SRCS ${KOISHI_PATH}/fcontext/fcontext.c ${KOISHI_PATH}/fcontext/fcontext.hpp) endif() +# Static LevelDB v1.23 is built with -fno-rtti by default. To inherit our logger from leveldb's logger, this file must be built with -fno-rtti to +set_source_files_properties(${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti") + list(APPEND REINDEXER_LIBRARIES reindexer) add_library(${TARGET} STATIC ${HDRS} ${SRCS} ${VENDORS}) add_definitions(-DREINDEX_CORE_BUILD=1) @@ -695,6 +702,7 @@ if (NOT WIN32) SET(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "server") SET(DIST_INCLUDE_FILES "tools/errors.h" "tools/serializer.h" "tools/varint.h" "tools/stringstools.h" "tools/customhash.h" "tools/assertrx.h" "tools/jsonstring.h" + "tools/verifying_updater.h" "core/reindexer.h" "core/type_consts.h" "core/item.h" "core/payload/payloadvalue.h" "core/payload/payloadiface.h" "core/indexopts.h" "core/namespacedef.h" "core/keyvalue/variant.h" "core/keyvalue/geometry.h" "core/sortingprioritiestable.h" "core/rdxcontext.h" "core/activity_context.h" "core/type_consts_helpers.h" "core/payload/fieldsset.h" "core/payload/payloadtype.h" @@ -702,8 +710,8 @@ if (NOT WIN32) "core/query/query.h" "core/query/queryentry.h" "core/queryresults/queryresults.h" "core/indexdef.h" "core/queryresults/aggregationresult.h" "core/queryresults/itemref.h" "core/namespace/stringsholder.h" "core/keyvalue/key_string.h" "core/key_value_type.h" "core/keyvalue/uuid.h" "core/expressiontree.h" "core/lsn.h" "core/cjson/tagspath.h" "core/cjson/ctag.h" - "estl/cow.h" "estl/overloaded.h" "estl/one_of.h" "estl/h_vector.h" "estl/mutex.h" "estl/intrusive_ptr.h" "estl/trivial_reverse_iterator.h" - "estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h" "estl/defines.h" + "estl/cow.h" "estl/overloaded.h" "estl/one_of.h" "estl/h_vector.h" "estl/mutex.h" "estl/intrusive_ptr.h" "estl/trivial_reverse_iterator.h" + "estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h" "estl/defines.h" "client/reindexer.h" "client/item.h" "client/reindexerconfig.h" "client/queryresults.h" "client/resultserializer.h" "client/internalrdxcontext.h" "client/transaction.h" "client/cororeindexer.h" "client/coroqueryresults.h" "client/corotransaction.h" diff --git a/cpp_src/client/cororeindexer.cc b/cpp_src/client/cororeindexer.cc index 4cb747cf5..4794f2ae6 100644 --- a/cpp_src/client/cororeindexer.cc +++ b/cpp_src/client/cororeindexer.cc @@ -33,7 +33,7 @@ CoroReindexer& CoroReindexer::operator=(CoroReindexer&& rdx) noexcept { Error CoroReindexer::Connect(const std::string& dsn, dynamic_loop& loop, const client::ConnectOpts& opts) { return impl_->Connect(dsn, loop, opts); } -Error CoroReindexer::Stop() { return impl_->Stop(); } +void CoroReindexer::Stop() { impl_->Stop(); } Error CoroReindexer::AddNamespace(const NamespaceDef& nsDef) { return impl_->AddNamespace(nsDef, ctx_); } Error CoroReindexer::OpenNamespace(std::string_view nsName, const StorageOpts& storage) { return impl_->OpenNamespace(nsName, ctx_, storage); diff --git a/cpp_src/client/cororeindexer.h b/cpp_src/client/cororeindexer.h index da211a6aa..25ef11f6f 100644 --- a/cpp_src/client/cororeindexer.h +++ b/cpp_src/client/cororeindexer.h @@ -41,12 +41,13 @@ class CoroReindexer { CoroReindexer &operator=(CoroReindexer &&) noexcept; /// Connect - connect to reindexer server - /// @param dsn - uri of server and database, like: `cproto://user@password:127.0.0.1:6534/dbname` + /// @param dsn - uri of server and database, like: `cproto://user@password:127.0.0.1:6534/dbname` or + /// `ucproto://user@password:/tmp/reindexer.sock:/dbname` /// @param loop - event loop for connections and coroutines handling /// @param opts - Connect options. May contaion any of
Error Connect(const std::string &dsn, dynamic_loop &loop, const client::ConnectOpts &opts = client::ConnectOpts()); /// Stop - shutdown connector - Error Stop(); + void Stop(); /// Open or create namespace /// @param nsName - Name of namespace /// @param opts - Storage options. Can be one of
diff --git a/cpp_src/client/cororpcclient.cc b/cpp_src/client/cororpcclient.cc index 91ab0114b..feb8cbb00 100644 --- a/cpp_src/client/cororpcclient.cc +++ b/cpp_src/client/cororpcclient.cc @@ -26,6 +26,7 @@ CoroRPCClient::CoroRPCClient(const ReindexerConfig& config) : config_(config) { CoroRPCClient::~CoroRPCClient() { Stop(); } Error CoroRPCClient::Connect(const std::string& dsn, ev::dynamic_loop& loop, const client::ConnectOpts& opts) { + using namespace std::string_view_literals; if (conn_.IsRunning()) { return Error(errLogic, "Client is already started"); } @@ -34,9 +35,15 @@ Error CoroRPCClient::Connect(const std::string& dsn, ev::dynamic_loop& loop, con if (!connectData.uri.parse(dsn)) { return Error(errParams, "%s is not valid uri", dsn); } - if (connectData.uri.scheme() != "cproto") { +#ifdef _WIN32 + if (connectData.uri.scheme() != "cproto"sv) { return Error(errParams, "Scheme must be cproto"); } +#else + if (connectData.uri.scheme() != "cproto"sv && connectData.uri.scheme() != "ucproto"sv) { + return Error(errParams, "Scheme must be either cproto or ucproto"); + } +#endif connectData.opts = cproto::CoroClientConnection::Options( config_.ConnectTimeout, config_.RequestTimeout, opts.IsCreateDBIfMissing(), opts.HasExpectedClusterID(), opts.ExpectedClusterID(), config_.ReconnectAttempts, config_.EnableCompression, config_.RequestDedicatedThread, config_.AppName); @@ -46,13 +53,12 @@ Error CoroRPCClient::Connect(const std::string& dsn, ev::dynamic_loop& loop, con return errOK; } -Error CoroRPCClient::Stop() { +void CoroRPCClient::Stop() { terminate_ = true; conn_.Stop(); resubWg_.wait(); loop_ = nullptr; terminate_ = false; - return errOK; } Error CoroRPCClient::AddNamespace(const NamespaceDef& nsDef, const InternalRdxContext& ctx) { @@ -236,7 +242,7 @@ Error CoroRPCClient::Delete(const Query& query, CoroQueryResults& result, const query.Serialize(ser); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = CoroQueryResults(&conn_, std::move(nsArray), 0, config_.FetchAmount, config_.RequestTimeout); @@ -257,7 +263,7 @@ Error CoroRPCClient::Update(const Query& query, CoroQueryResults& result, const query.Serialize(ser); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = CoroQueryResults(&conn_, std::move(nsArray), 0, config_.FetchAmount, config_.RequestTimeout); @@ -322,7 +328,7 @@ Error CoroRPCClient::selectImpl(const Query& query, CoroQueryResults& result, se } NsArray nsArray; query.Serialize(qser); - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); h_vector vers; for (auto& ns : nsArray) { vers.push_back(ns->tagsMatcher_.version() ^ ns->tagsMatcher_.stateToken()); diff --git a/cpp_src/client/cororpcclient.h b/cpp_src/client/cororpcclient.h index 8c217aa3d..50278e8fd 100644 --- a/cpp_src/client/cororpcclient.h +++ b/cpp_src/client/cororpcclient.h @@ -36,7 +36,7 @@ class CoroRPCClient { ~CoroRPCClient(); Error Connect(const std::string &dsn, ev::dynamic_loop &loop, const client::ConnectOpts &opts); - Error Stop(); + void Stop(); Error OpenNamespace(std::string_view nsName, const InternalRdxContext &ctx, const StorageOpts &opts = StorageOpts().Enabled().CreateIfMissing()); diff --git a/cpp_src/client/item.h b/cpp_src/client/item.h index 3f1bbd684..618398a5c 100644 --- a/cpp_src/client/item.h +++ b/cpp_src/client/item.h @@ -60,13 +60,13 @@ class Item { /// Get status of item /// @return data slice with JSON. Returned slice is allocated in temporary Item's buffer, and can be invalidated by any next operation /// with Item - Error Status() { return status_; } + Error Status() const noexcept { return status_; } /// Get internal ID of item /// @return ID of item int GetID() const noexcept { return id_; } - /// Get internal version of item - /// @return version of item - int NumFields(); + /// Get count of indexed fields + /// @return count of indexed fields + int NumFields() const noexcept; /// Set additional percepts for modify operation /// @param precepts - strings in format "fieldName=Func()" void SetPrecepts(const std::vector &precepts); diff --git a/cpp_src/client/itemimpl.cc b/cpp_src/client/itemimpl.cc index 3d33bf1db..ef15c097d 100644 --- a/cpp_src/client/itemimpl.cc +++ b/cpp_src/client/itemimpl.cc @@ -46,9 +46,9 @@ void ItemImpl::FromCJSON(std::string_view slice) { } Payload pl = GetPayload(); - CJsonDecoder decoder(tagsMatcher_); + CJsonDecoder decoder(tagsMatcher_, holder_); ser_.Reset(); - decoder.Decode(pl, rdser, ser_); + decoder.Decode<>(pl, rdser, ser_); if (!rdser.Eof() && rdser.Pos() != tmOffset) { throw Error(errParseJson, "Internal error - left unparsed data %d", rdser.Pos()); diff --git a/cpp_src/client/queryresults.cc b/cpp_src/client/queryresults.cc index 7f0b070b1..83b1755c3 100644 --- a/cpp_src/client/queryresults.cc +++ b/cpp_src/client/queryresults.cc @@ -114,8 +114,6 @@ void QueryResults::fetchNextResults() { rawResult_.assign(rawResult.begin() + ser.Pos(), rawResult.end()); } -QueryResults::~QueryResults() {} - h_vector QueryResults::GetNamespaces() const { h_vector ret; ret.reserve(nsArray_.size()); @@ -310,8 +308,5 @@ QueryResults::Iterator &QueryResults::Iterator::operator++() { return *this; } -bool QueryResults::Iterator::operator!=(const Iterator &other) const { return idx_ != other.idx_; } -bool QueryResults::Iterator::operator==(const Iterator &other) const { return idx_ == other.idx_; } - } // namespace client } // namespace reindexer diff --git a/cpp_src/client/queryresults.h b/cpp_src/client/queryresults.h index 97c2a1835..e8b059568 100644 --- a/cpp_src/client/queryresults.h +++ b/cpp_src/client/queryresults.h @@ -26,7 +26,7 @@ class QueryResults { QueryResults(int fetchFlags = 0); QueryResults(const QueryResults&) = delete; QueryResults(QueryResults&&) noexcept; - ~QueryResults(); + ~QueryResults() = default; QueryResults& operator=(const QueryResults&) = delete; QueryResults& operator=(QueryResults&& obj) noexcept; @@ -40,10 +40,10 @@ class QueryResults { bool IsRaw(); std::string_view GetRaw(); Iterator& operator++(); - Error Status() { return qr_->status_; } - bool operator!=(const Iterator&) const; - bool operator==(const Iterator&) const; - Iterator& operator*() { return *this; } + Error Status() const noexcept { return qr_->status_; } + bool operator==(const Iterator& other) const noexcept { return idx_ == other.idx_; } + bool operator!=(const Iterator& other) const noexcept { return !operator==(other); } + Iterator& operator*() noexcept { return *this; } void readNext(); void getJSONFromCJSON(std::string_view cjson, WrSerializer& wrser, bool withHdrLen = true); @@ -55,13 +55,15 @@ class QueryResults { Iterator begin() const { return Iterator{this, 0, 0, 0, {}}; } Iterator end() const { return Iterator{this, queryParams_.qcount, 0, 0, {}}; } - size_t Count() const { return queryParams_.qcount; } - int TotalCount() const { return queryParams_.totalcount; } - bool HaveRank() const { return queryParams_.flags & kResultsWithRank; } - bool NeedOutputRank() const { return queryParams_.flags & kResultsNeedOutputRank; } - const std::string& GetExplainResults() const { return queryParams_.explainResults; } - const std::vector& GetAggregationResults() const { return queryParams_.aggResults; } - Error Status() { return status_; } + size_t Count() const noexcept { return queryParams_.qcount; } + int TotalCount() const noexcept { return queryParams_.totalcount; } + bool HaveRank() const noexcept { return queryParams_.flags & kResultsWithRank; } + bool NeedOutputRank() const noexcept { return queryParams_.flags & kResultsNeedOutputRank; } + const std::string& GetExplainResults() const& noexcept { return queryParams_.explainResults; } + const std::string& GetExplainResults() const&& = delete; + const std::vector& GetAggregationResults() const& noexcept { return queryParams_.aggResults; } + const std::vector& GetAggregationResults() const&& = delete; + Error Status() const noexcept { return status_; } h_vector GetNamespaces() const; bool IsCacheEnabled() const { return queryParams_.flags & kResultsWithItemID; } diff --git a/cpp_src/client/reindexer.cc b/cpp_src/client/reindexer.cc index e28041713..248b25cb0 100644 --- a/cpp_src/client/reindexer.cc +++ b/cpp_src/client/reindexer.cc @@ -29,7 +29,7 @@ Error Reindexer::Connect(const std::string& dsn, const client::ConnectOpts& opts Error Reindexer::Connect(const std::vector>& connectData) { return impl_->Connect(connectData); } -Error Reindexer::Stop() { return impl_->Stop(); } +void Reindexer::Stop() { impl_->Stop(); } Error Reindexer::AddNamespace(const NamespaceDef& nsDef) { return impl_->AddNamespace(nsDef, ctx_); } Error Reindexer::OpenNamespace(std::string_view nsName, const StorageOpts& storage) { return impl_->OpenNamespace(nsName, ctx_, storage); } Error Reindexer::DropNamespace(std::string_view nsName) { return impl_->DropNamespace(nsName, ctx_); } diff --git a/cpp_src/client/reindexer.h b/cpp_src/client/reindexer.h index 705de78e3..a8a3a8492 100644 --- a/cpp_src/client/reindexer.h +++ b/cpp_src/client/reindexer.h @@ -41,14 +41,15 @@ class Reindexer { Reindexer &operator=(Reindexer &&) noexcept; /// Connect - connect to reindexer server - /// @param dsn - uri of server and database, like: `cproto://user@password:127.0.0.1:6534/dbname` + /// @param dsn - uri of server and database, like: `cproto://user@password:127.0.0.1:6534/dbname` or + /// `ucproto://user@password:/tmp/reindexer.sock:/dbname` /// @param opts - Connect options. May contaion any of
Error Connect(const std::string &dsn, const client::ConnectOpts &opts = client::ConnectOpts()); /// Connect - connect to reindexer server /// @param connectData - list of server dsn + it's ConnectOpts Error Connect(const std::vector> &connectData); /// Stop - shutdown connector - Error Stop(); + void Stop(); /// Open or create namespace /// @param nsName - Name of namespace /// @param opts - Storage options. Can be one of
diff --git a/cpp_src/client/rpcclient.cc b/cpp_src/client/rpcclient.cc index 29b6f5ccc..f0f4b4b72 100644 --- a/cpp_src/client/rpcclient.cc +++ b/cpp_src/client/rpcclient.cc @@ -76,8 +76,8 @@ Error RPCClient::Connect(const std::vectorSetTerminateFlag(); if (connections_[i]->PendingCompletions()) { doTerminate = false; @@ -383,7 +381,7 @@ Error RPCClient::Delete(const Query& query, QueryResults& result, const Internal auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -410,7 +408,7 @@ Error RPCClient::Update(const Query& query, QueryResults& result, const Internal auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -486,7 +484,7 @@ Error RPCClient::selectImpl(const Query& query, QueryResults& result, cproto::Cl } NsArray nsArray; query.Serialize(qser); - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); h_vector vers; for (auto& ns : nsArray) { shared_lock lck(ns->lck_); diff --git a/cpp_src/client/rpcclient.h b/cpp_src/client/rpcclient.h index 238b16a12..eefb2cf03 100644 --- a/cpp_src/client/rpcclient.h +++ b/cpp_src/client/rpcclient.h @@ -41,7 +41,7 @@ class RPCClient { Error Connect(const std::string &dsn, const client::ConnectOpts &opts); Error Connect(const std::vector> &connectData); - Error Stop(); + void Stop(); Error OpenNamespace(std::string_view nsName, const InternalRdxContext &ctx, const StorageOpts &opts = StorageOpts().Enabled().CreateIfMissing()); diff --git a/cpp_src/client/rpcclientmock.cc b/cpp_src/client/rpcclientmock.cc index f2cb5635e..5c0d4b6ef 100644 --- a/cpp_src/client/rpcclientmock.cc +++ b/cpp_src/client/rpcclientmock.cc @@ -37,7 +37,7 @@ Error RPCClientMock::Delete(const Query& query, QueryResults& result, const Inte auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -69,7 +69,7 @@ Error RPCClientMock::Update(const Query& query, QueryResults& result, const Inte auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -308,7 +308,7 @@ Error RPCClientMock::selectImpl(const Query& query, QueryResults& result, cproto NsArray nsArray; query.Serialize(qser); - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); h_vector vers; for (auto& ns : nsArray) { shared_lock lck(ns->lck_); diff --git a/cpp_src/client/synccororeindexer.h b/cpp_src/client/synccororeindexer.h index 79257bb86..24e74dc81 100644 --- a/cpp_src/client/synccororeindexer.h +++ b/cpp_src/client/synccororeindexer.h @@ -25,7 +25,8 @@ class SyncCoroReindexer { SyncCoroReindexer &operator=(SyncCoroReindexer &&rdx) noexcept; /// Connect - connect to reindexer server - /// @param dsn - uri of server and database, like: `cproto://user@password:127.0.0.1:6534/dbname` + /// @param dsn - uri of server and database, like: `cproto://user@password:127.0.0.1:6534/dbname` or + /// `ucproto://user@password:/tmp/reindexer.sock:/dbname` /// @param opts - Connect options. May contaion any of
Error Connect(const std::string &dsn, const client::ConnectOpts &opts = client::ConnectOpts()); /// Stop - shutdown connector diff --git a/cpp_src/client/synccororeindexerimpl.cc b/cpp_src/client/synccororeindexerimpl.cc index 1ca2b1d1b..2d005b291 100644 --- a/cpp_src/client/synccororeindexerimpl.cc +++ b/cpp_src/client/synccororeindexerimpl.cc @@ -108,7 +108,7 @@ Error SyncCoroReindexerImpl::GetMeta(std::string_view nsName, const std::string } Error SyncCoroReindexerImpl::PutMeta(std::string_view nsName, const std::string &key, std::string_view data, const InternalRdxContext &ctx) { - return sendCommand(DbCmdPutMeta, std::forward(nsName), key, data, ctx); + return sendCommand(DbCmdPutMeta, std::forward(nsName), key, std::forward(data), ctx); } Error SyncCoroReindexerImpl::EnumMeta(std::string_view nsName, std::vector &keys, const InternalRdxContext &ctx) { return sendCommand(DbCmdEnumMeta, std::forward(nsName), keys, ctx); diff --git a/cpp_src/cmd/reindexer_server/test/get_last_rx_version.py b/cpp_src/cmd/reindexer_server/test/get_last_rx_version.py new file mode 100644 index 000000000..8007b8201 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/get_last_rx_version.py @@ -0,0 +1,28 @@ +import argparse +import re + +import requests +from packaging.version import parse + + +URL = "http://repo.restream.ru/itv-api-ng/7/x86_64/" + +parser = argparse.ArgumentParser(description='Version') +parser.add_argument('-v', '--version', default="3") +args = parser.parse_args() + +version = args.version +if version == "3": + name = ">reindexer-server-" +elif version == "4": + name = ">reindexer-4-server-" +else: + raise ValueError(f"Version {version} is invalid") + +r = requests.get(URL) +res = r.text +res_list = re.findall(f'{name}.*.rpm', res) +versions_list = [(i[1:], parse(i[len(name):-11])) for i in res_list] +versions_list.sort(key=lambda x: x[1]) + +print(versions_list[-1][0]) diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100755 index 000000000..d189d3841 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +echo "## choose latest release rpm file" +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 3) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 4) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip; +unzip -o big.zip # unzips into mydb_big.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"replication":{"data_hash":24651210926,"data_count":3}}, +{"replication":{"data_hash":6252344969,"data_count":1}}, +{"replication":{"data_hash":37734732881,"data_count":28}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":1024095024522,"data_count":1145}}, +{"replication":{"data_hash":8373644068,"data_count":1315}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":7404222244,"data_count":97}}, +{"replication":{"data_hash":94132837196,"data_count":4}}, +{"replication":{"data_hash":1896088071,"data_count":2}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":-672103903,"data_count":33538}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":6833710705,"data_count":1}}, +{"replication":{"data_hash":5858155773472,"data_count":4500}}, +{"replication":{"data_hash":-473221280268823592,"data_count":65448}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":8288213744,"data_count":3}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":354171024786967,"data_count":3941}}, +{"replication":{"data_hash":-6520334670,"data_count":35886}}, +{"replication":{"data_hash":112772074632,"data_count":281}}, +{"replication":{"data_hash":-12679568198538,"data_count":1623116}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc index 8509155f5..77c3abb70 100644 --- a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc @@ -47,19 +47,18 @@ Error CommandsExecutor::Run(const std::string& } template -void CommandsExecutor::GetSuggestions(const std::string& input, std::vector& suggestions) { +Error CommandsExecutor::GetSuggestions(const std::string& input, std::vector& suggestions) { OutParamCommand> cmd( - [this, &input](std::vector& suggestions) { - getSuggestions(input, suggestions); - return errOK; - }, - suggestions); - execCommand(cmd); + [this, &input](std::vector& suggestions) { return getSuggestions(input, suggestions); }, suggestions); + return execCommand(cmd); } template Error CommandsExecutor::Stop() { - GenericCommand cmd([this] { return stop(true); }); + GenericCommand cmd([this] { + stop(true); + return Error{}; + }); auto err = execCommand(cmd); if (err.ok() && executorThr_.joinable()) { executorThr_.join(); @@ -277,11 +276,26 @@ Error CommandsExecutor::runImpl(const std::string& dsn, Args&&... a template std::string CommandsExecutor::getCurrentDsn(bool withPath) const { + using namespace std::string_view_literals; std::string dsn(uri_.scheme() + "://"); if (!uri_.password().empty() && !uri_.username().empty()) { - dsn += uri_.username() + ":" + uri_.password() + "@"; + dsn += uri_.username() + ':' + uri_.password() + '@'; + } + if (uri_.scheme() == "ucproto"sv) { + std::vector pathParts; + reindexer::split(std::string_view(uri_.path()), ":", true, pathParts); + std::string_view dbName; + if (pathParts.size() >= 2) { + dbName = pathParts.back(); + } + if (dbName.size()) { + dsn += uri_.path().substr(0, uri_.path().size() - dbName.size() - 1) + ':' + (withPath ? uri_.path() : "/"); + } else { + dsn += uri_.path() + ':' + (withPath ? uri_.path() : "/"); + } + } else { + dsn += uri_.hostname() + ':' + uri_.port() + (withPath ? uri_.path() : "/"); } - dsn += uri_.hostname() + ":" + uri_.port() + (withPath ? uri_.path() : "/"); return dsn; } @@ -475,15 +489,14 @@ Error CommandsExecutor::processImpl(const std::string& command) noe } template <> -Error CommandsExecutor::stop(bool terminate) { +void CommandsExecutor::stop(bool terminate) { if (terminate) { stopCh_.close(); } - return Error(); } template <> -Error CommandsExecutor::stop(bool terminate) { +void CommandsExecutor::stop(bool terminate) { if (terminate) { stopCh_.close(); } @@ -491,11 +504,17 @@ Error CommandsExecutor::stop(bool terminate) { } template -void CommandsExecutor::getSuggestions(const std::string& input, std::vector& suggestions) { - if (!input.empty() && input[0] != '\\') db().GetSqlSuggestions(input, input.length() - 1, suggestions); +Error CommandsExecutor::getSuggestions(const std::string& input, std::vector& suggestions) { + if (!input.empty() && input[0] != '\\') { + auto err = db().GetSqlSuggestions(input, input.length() - 1, suggestions); + if (!err.ok()) { + return err; + } + } if (suggestions.empty()) { addCommandsSuggestions(input, suggestions); } + return {}; } template @@ -761,7 +780,8 @@ Error CommandsExecutor::commandDump(const std::string& command) { return Error(errCanceled, "Canceled"); } wrser << "\\UPSERT " << reindexer::escapeString(nsDef.name) << ' '; - it.GetJSON(wrser, false); + err = it.GetJSON(wrser, false); + if (!err.ok()) return err; wrser << '\n'; if (wrser.Len() > 0x100000) { output_() << wrser.Slice(); @@ -847,9 +867,11 @@ Error CommandsExecutor::commandMeta(const std::string& command) { auto nsName = reindexer::unescapeString(parser.NextToken()); std::vector allMeta; auto err = db().EnumMeta(nsName, allMeta); + if (!err.ok()) return err; for (auto& metaKey : allMeta) { std::string metaData; - db().GetMeta(nsName, metaKey, metaData); + err = db().GetMeta(nsName, metaKey, metaData); + if (!err.ok()) return err; output_() << metaKey << " = " << metaData << std::endl; } return err; @@ -917,15 +939,18 @@ Error CommandsExecutor::commandBench(const std::string& command) { LineParser parser(command); parser.NextToken(); - int benchTime = reindexer::stoi(parser.NextToken()); - if (benchTime == 0) benchTime = kBenchDefaultTime; + const std::string_view benchTimeToken = parser.NextToken(); + const int benchTime = benchTimeToken.empty() ? kBenchDefaultTime : reindexer::stoi(benchTimeToken); - db().DropNamespace(kBenchNamespace); + auto err = db().DropNamespace(kBenchNamespace); + if (!err.ok() && err.code() != errNotFound) { + return err; + } NamespaceDef nsDef(kBenchNamespace); nsDef.AddIndex("id", "hash", "int", IndexOpts().PK()); - auto err = db().AddNamespace(nsDef); + err = db().AddNamespace(nsDef); if (!err.ok()) return err; output_() << "Seeding " << kBenchItemsCount << " documents to bench namespace..." << std::endl; @@ -1002,31 +1027,30 @@ Error CommandsExecutor::commandSubscribe(const std::string& command template <> Error CommandsExecutor::commandProcessDatabases(const std::string& command) { + using namespace std::string_view_literals; LineParser parser(command); parser.NextToken(); std::string_view subCommand = parser.NextToken(); - assertrx(uri_.scheme() == "cproto"); - if (subCommand == "list") { + assertrx(uri_.scheme() == "cproto"sv || uri_.scheme() == "ucproto"sv); + if (subCommand == "list"sv) { std::vector dbList; Error err = getAvailableDatabases(dbList); if (!err.ok()) return err; for (const std::string& dbName : dbList) output_() << dbName << std::endl; return Error(); - } else if (subCommand == "use") { + } else if (subCommand == "use"sv) { std::string currentDsn = getCurrentDsn() + std::string(parser.NextToken()); - Error err = stop(false); - if (!err.ok()) return err; - err = db().Connect(currentDsn, loop_); + stop(false); + auto err = db().Connect(currentDsn, loop_); if (err.ok()) err = db().Status(); if (err.ok()) output_() << "Succesfully connected to " << currentDsn << std::endl; return err; - } else if (subCommand == "create") { + } else if (subCommand == "create"sv) { auto dbName = parser.NextToken(); std::string currentDsn = getCurrentDsn() + std::string(dbName); - Error err = stop(false); - if (!err.ok()) return err; + stop(false); output_() << "Creating database '" << dbName << "'" << std::endl; - err = db().Connect(currentDsn, loop_, reindexer::client::ConnectOpts().CreateDBIfMissing()); + auto err = db().Connect(currentDsn, loop_, reindexer::client::ConnectOpts().CreateDBIfMissing()); if (!err.ok()) { std::cerr << "Error on database '" << dbName << "' creation" << std::endl; return err; @@ -1145,11 +1169,14 @@ std::function CommandsExecutor Error Run(const std::string& dsn, const Args&... args); - void GetSuggestions(const std::string& input, std::vector& suggestions); + Error GetSuggestions(const std::string& input, std::vector& suggestions); Error Stop(); Error Process(const std::string& command); Error FromFile(std::istream& in); @@ -78,8 +78,8 @@ class CommandsExecutor : public reindexer::IUpdatesObserver { std::vector& suggestions); Error processImpl(const std::string& command) noexcept; - Error stop(bool terminate); - void getSuggestions(const std::string& input, std::vector& suggestions); + void stop(bool terminate); + Error getSuggestions(const std::string& input, std::vector& suggestions); Error commandSelect(const std::string& command); Error commandUpsert(const std::string& command); Error commandUpdateSQL(const std::string& command); diff --git a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc index 17ce39931..f084ea387 100644 --- a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc @@ -3,8 +3,8 @@ #include #include "client/cororeindexer.h" #include "core/reindexer.h" -#include "tableviewscroller.h" #include "tools/fsops.h" +#include "tools/terminalutils.h" namespace reindexer_tool { @@ -30,9 +30,13 @@ template void CommandsProcessor::setCompletionCallback(T& rx, void (T::*set_completion_callback)(new_v_callback_t const&)) { (rx.*set_completion_callback)([this](std::string const& input, int) -> replxx::Replxx::completions_t { std::vector completions; - executor_.GetSuggestions(input, completions); + const auto err = executor_.GetSuggestions(input, completions); replxx::Replxx::completions_t result; - for (const std::string& suggestion : completions) result.emplace_back(suggestion); + if (err.ok()) { + for (const std::string& suggestion : completions) { + result.emplace_back(suggestion); + } + } return result; }); } @@ -43,7 +47,8 @@ void CommandsProcessor::setCompletionCallback(T& rx, void (T::*set_ (rx.*set_completion_callback)( [this](std::string const& input, int, void*) -> replxx::Replxx::completions_t { std::vector completions; - executor_.GetSuggestions(input, completions); + const auto err = executor_.GetSuggestions(input, completions); + if (!err.ok()) return {}; return completions; }, nullptr); diff --git a/cpp_src/cmd/reindexer_tool/readme.md b/cpp_src/cmd/reindexer_tool/readme.md index e7a71e6c2..19be92a3a 100644 --- a/cpp_src/cmd/reindexer_tool/readme.md +++ b/cpp_src/cmd/reindexer_tool/readme.md @@ -21,7 +21,7 @@ Reindexer command line tool is an client utility to work with database. reindexer_tool {OPTIONS} Options - -d[DSN], --dsn=[DSN] DSN to 'reindexer', like 'cproto://127.0.0.1:6534/dbname' or 'builtin:///var/lib/reindexer/dbname' + -d[DSN], --dsn=[DSN] DSN to 'reindexer', like 'cproto://127.0.0.1:6534/dbname', 'builtin:///var/lib/reindexer/dbname' or `ucproto://user@password:/tmp/reindexer.sock:/dbname` -f[FILENAME], --filename=[FILENAME] execute commands from file, then exit -c[COMMAND], --command=[COMMAND] run only single command (SQL or internal) and exit -o[FILENAME], --output=[FILENAME] send query results to file diff --git a/cpp_src/cmd/reindexer_tool/reindexer_tool.cc b/cpp_src/cmd/reindexer_tool/reindexer_tool.cc index cb5be7eb0..fdf8ff498 100644 --- a/cpp_src/cmd/reindexer_tool/reindexer_tool.cc +++ b/cpp_src/cmd/reindexer_tool/reindexer_tool.cc @@ -56,9 +56,16 @@ int main(int argc, char* argv[]) { args::HelpFlag help(parser, "help", "show this message", {'h', "help"}); args::Group progOptions("options"); +#ifdef _WIN32 args::ValueFlag dbDsn(progOptions, "DSN", "DSN to 'reindexer'. Can be 'cproto://:/' or 'builtin://'", {'d', "dsn"}, "", Options::Single | Options::Global); +#else // _WIN32 + args::ValueFlag dbDsn( + progOptions, "DSN", + "DSN to 'reindexer'. Can be 'cproto://:/', 'builtin://' or 'ucproto://:/'", + {'d', "dsn"}, "", Options::Single | Options::Global); +#endif // _WIN32 args::ValueFlag fileName(progOptions, "FILENAME", "execute commands from file, then exit", {'f', "filename"}, "", Options::Single | Options::Global); args::ValueFlag command(progOptions, "COMMAND", "run only single command (SQL or internal) and exit'", {'c', "command"}, @@ -104,6 +111,8 @@ int main(int argc, char* argv[]) { signal(SIGPIPE, SIG_IGN); #endif + using namespace std::string_view_literals; + using reindexer::checkIfStartsWithCS; std::string db; if (dsn.empty()) { db = args::get(dbName); @@ -111,7 +120,15 @@ int main(int argc, char* argv[]) { std::cerr << "Error: --dsn either database name should be set as a first argument" << std::endl; return 2; } - if (db.substr(0, 9) == "cproto://" || db.substr(0, 10) == "builtin://") { + + if (checkIfStartsWithCS("cproto://"sv, db) || checkIfStartsWithCS("ucproto://"sv, db) || checkIfStartsWithCS("builtin://"sv, db)) { +#ifdef _WIN32 + if (checkIfStartsWithCS("ucproto://"sv, db) == 0) { + std::cerr << "Invalid DSN: ucproto:// is not supported on the Windows platform. Use cproto:// or builtin:// instead" + << std::endl; + return 2; + } +#endif // _WIN32 dsn = db; } else { dsn = "cproto://reindexer:reindexer@127.0.0.1:6534/" + db; @@ -131,7 +148,14 @@ int main(int argc, char* argv[]) { std::cout << "Reindexer command line tool version " << REINDEX_VERSION << std::endl; } - if (dsn.compare(0, 9, "cproto://") == 0) { + if (checkIfStartsWithCS("cproto://"sv, dsn) || checkIfStartsWithCS("ucproto://"sv, dsn)) { +#ifdef _WIN32 + if (checkIfStartsWithCS("ucproto://"sv, dsn)) { + std::cerr << "Invalid DSN: ucproto:// is not supported on the Windows platform. Use cproto:// or builtin:// instead" + << std::endl; + return 2; + } +#endif // _WIN32 reindexer::client::ReindexerConfig config; config.EnableCompression = true; config.AppName = args::get(appName); @@ -139,13 +163,17 @@ int main(int argc, char* argv[]) { args::get(connThreads), config); err = commandsProcessor.Connect(dsn, reindexer::client::ConnectOpts().CreateDBIfMissing(createDBF && args::get(createDBF))); if (err.ok()) ok = commandsProcessor.Run(args::get(command)); - } else if (dsn.compare(0, 10, "builtin://") == 0) { + } else if (checkIfStartsWithCS("builtin://"sv, dsn)) { reindexer::Reindexer db; CommandsProcessor commandsProcessor(args::get(outFileName), args::get(fileName), args::get(connThreads)); err = commandsProcessor.Connect(dsn, ConnectOpts().DisableReplication()); if (err.ok()) ok = commandsProcessor.Run(args::get(command)); } else { +#ifdef _WIN32 std::cerr << "Invalid DSN format: " << dsn << " Must begin from cproto:// or builtin://" << std::endl; +#else // _WIN32 + std::cerr << "Invalid DSN format: " << dsn << " Must begin from cproto://, ucproto:// or builtin://" << std::endl; +#endif // _WIN32 } if (!err.ok()) { std::cerr << "ERROR: " << err.what() << std::endl; diff --git a/cpp_src/core/cjson/cjsondecoder.cc b/cpp_src/core/cjson/cjsondecoder.cc index b74b2bcba..53a5162cf 100644 --- a/cpp_src/core/cjson/cjsondecoder.cc +++ b/cpp_src/core/cjson/cjsondecoder.cc @@ -7,71 +7,57 @@ namespace reindexer { -bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrser, bool match) { +template +bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrser, FilterT filter, RecoderT recoder, TagOptT) { const ctag tag = rdser.GetCTag(); TagType tagType = tag.Type(); if (tagType == TAG_END) { wrser.PutCTag(kCTagEnd); return false; } - const int tagName = tag.Name(); - if (tagName) { - // Check + int tagName = 0; + if constexpr (std::is_same_v) { + tagName = tag.Name(); + assertrx_dbg(tagName); + // Check if tag exists (void)tagsMatcher_.tag2name(tagName); tagsPath_.emplace_back(tagName); } + if rx_unlikely (tag.Field() >= 0) { - throw Error(errLogic, "Reference tag was found in transport CJSON for field %d[%s] in ns [%s]", tag.Field(), - tagsMatcher_.tag2name(tagName), pl.Type().Name()); + throwTagReferenceError(tag, pl); } const int field = tagsMatcher_.tags2field(tagsPath_.data(), tagsPath_.size()); - - if (filter_) { - if (field >= 0) { - match = filter_->contains(field); - } else { - match = match && filter_->match(tagsPath_); - } - } - Recoder *recoder{nullptr}; - if (recoder_) { - if (field >= 0) { - if (recoder_->Match(field)) { - recoder = recoder_; - } - } else { - if (recoder_->Match(tagsPath_)) { - recoder = recoder_; - } - } - } - if (recoder) { - tagType = recoder->Type(tagType); - } if (field >= 0) { + const bool match = filter.contains(field); if (match) { + tagType = recoder.RegisterTagType(tagType, field); if (tagType == TAG_NULL) { - objectScalarIndexes_.set(field); wrser.PutCTag(ctag{TAG_NULL, tagName}); - } else if (recoder) { - recoder->Recode(rdser, pl, tagName, wrser); + } else if (recoder.Recode(rdser, pl, tagName, wrser)) { + // No more actions needed after recoding } else { const auto &fieldRef{pl.Type().Field(field)}; const KeyValueType fieldType{fieldRef.Type()}; if (tagType == TAG_ARRAY) { if rx_unlikely (!fieldRef.IsArray()) { - throw Error(errLogic, "Error parsing cjson field '%s' - got array, expected scalar %s", fieldRef.Name(), - fieldType.Name()); + throwUnexpectedArrayError(fieldRef); } const carraytag atag = rdser.GetCArrayTag(); const auto count = atag.Count(); const int ofs = pl.ResizeArray(field, count, true); const TagType atagType = atag.Type(); - for (size_t i = 0; i < count; ++i) { - const TagType type = atagType != TAG_OBJECT ? atagType : rdser.GetCTag().Type(); - pl.Set(field, ofs + i, cjsonValueToVariant(type, rdser, fieldType)); + if (atagType != TAG_OBJECT) { + for (size_t i = 0; i < count; ++i) { + pl.Set(field, ofs + i, cjsonValueToVariant(atagType, rdser, fieldType)); + } + } else { + for (size_t i = 0; i < count; ++i) { + pl.Set(field, ofs + i, cjsonValueToVariant(rdser.GetCTag().Type(), rdser, fieldType)); + } } + wrser.PutCTag(ctag{TAG_ARRAY, tagName, field}); wrser.PutVarUint(count); } else { @@ -93,44 +79,76 @@ bool CJsonDecoder::decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrs skipCjsonTag(tag, rdser); } } else { - wrser.PutCTag(ctag{tagType, tagName, field}); - if (tagType == TAG_OBJECT) { - while (decodeCJson(pl, rdser, wrser, match)) - ; - } else if (!match) { - skipCjsonTag(tag, rdser); - } else if (recoder) { - recoder->Recode(rdser, wrser); - } else if (tagType == TAG_ARRAY) { - const carraytag atag = rdser.GetCArrayTag(); - wrser.PutCArrayTag(atag); - const auto count = atag.Count(); - const TagType atagType = atag.Type(); - CounterGuardIR32 g(arrayLevel_); - for (size_t i = 0; i < count; ++i) { - switch (atagType) { - case TAG_OBJECT: - decodeCJson(pl, rdser, wrser, match); - break; - case TAG_VARINT: - case TAG_NULL: - case TAG_BOOL: - case TAG_STRING: - case TAG_END: - case TAG_DOUBLE: - case TAG_ARRAY: - case TAG_UUID: + const bool match = filter.match(tagsPath_); + if (match) { + tagType = recoder.RegisterTagType(tagType, tagsPath_); + wrser.PutCTag(ctag{tagType, tagName, field}); + if (tagType == TAG_OBJECT) { + while (decodeCJson(pl, rdser, wrser, filter.MakeCleanCopy(), recoder.MakeCleanCopy(), NamedTagOpt{})) + ; + } else if (recoder.Recode(rdser, wrser)) { + // No more actions needed after recoding + } else if (tagType == TAG_ARRAY) { + const carraytag atag = rdser.GetCArrayTag(); + wrser.PutCArrayTag(atag); + const auto count = atag.Count(); + const TagType atagType = atag.Type(); + CounterGuardIR32 g(arrayLevel_); + if (atagType == TAG_OBJECT) { + for (size_t i = 0; i < count; ++i) { + decodeCJson(pl, rdser, wrser, filter.MakeCleanCopy(), recoder.MakeCleanCopy(), NamelessTagOpt{}); + } + } else { + for (size_t i = 0; i < count; ++i) { copyCJsonValue(atagType, rdser, wrser); - break; + } } + } else { + copyCJsonValue(tagType, rdser, wrser); } + } else if (tagType != TAG_OBJECT) { + // !match + skipCjsonTag(tag, rdser); } else { - copyCJsonValue(tagType, rdser, wrser); + // !match + wrser.PutCTag(ctag{tagType, tagName, field}); + while (decodeCJson(pl, rdser, wrser, filter.MakeSkipFilter(), recoder.MakeCleanCopy(), NamedTagOpt{})) + ; } } - if (tagName) tagsPath_.pop_back(); + if constexpr (std::is_same_v) { + tagsPath_.pop_back(); + } + return true; } +[[nodiscard]] Variant CJsonDecoder::cjsonValueToVariant(TagType tagType, Serializer &rdser, KeyValueType fieldType) { + if (fieldType.Is() && tagType != TagType::TAG_STRING) { + storage_.emplace_back(rdser.GetRawVariant(KeyValueType{tagType}).As()); + return Variant(p_string(&storage_.back()), false); + } else { + return reindexer::cjsonValueToVariant(tagType, rdser, fieldType); + } +} + +RX_NO_INLINE void CJsonDecoder::throwTagReferenceError(ctag tag, const Payload &pl) { + throw Error(errLogic, "Reference tag was found in transport CJSON for field %d[%s] in ns [%s]", tag.Field(), + tagsMatcher_.tag2name(tag.Name()), pl.Type().Name()); +} + +RX_NO_INLINE void CJsonDecoder::throwUnexpectedArrayError(const PayloadFieldType &fieldRef) { + throw Error(errLogic, "Error parsing cjson field '%s' - got array, expected scalar %s", fieldRef.Name(), fieldRef.Type().Name()); +} + +template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::DummyFilter, CJsonDecoder::DummyRecoder, CJsonDecoder::NamelessTagOpt); +template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::DummyFilter, CJsonDecoder::DefaultRecoder, CJsonDecoder::NamelessTagOpt); +template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::RestrictingFilter, CJsonDecoder::DummyRecoder, CJsonDecoder::NamelessTagOpt); +template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::RestrictingFilter, CJsonDecoder::DefaultRecoder, CJsonDecoder::NamelessTagOpt); + } // namespace reindexer diff --git a/cpp_src/core/cjson/cjsondecoder.h b/cpp_src/core/cjson/cjsondecoder.h index a128fe487..9835c4039 100644 --- a/cpp_src/core/cjson/cjsondecoder.h +++ b/cpp_src/core/cjson/cjsondecoder.h @@ -1,5 +1,6 @@ #pragma once +#include #include "core/payload/payloadiface.h" namespace reindexer { @@ -20,25 +21,157 @@ class Recoder { class CJsonDecoder { public: - CJsonDecoder(TagsMatcher &tagsMatcher) noexcept : tagsMatcher_(tagsMatcher), filter_(nullptr) {} - CJsonDecoder(TagsMatcher &tagsMatcher, const FieldsSet *filter, Recoder *recoder) noexcept - : tagsMatcher_(tagsMatcher), filter_(filter), recoder_(recoder) {} + explicit CJsonDecoder(TagsMatcher &tagsMatcher, std::deque &storage) noexcept + : tagsMatcher_(tagsMatcher), storage_(storage) {} + class SkipFilter { + public: + SkipFilter MakeCleanCopy() const noexcept { return SkipFilter(); } + SkipFilter MakeSkipFilter() const noexcept { return SkipFilter(); } - void Decode(Payload &pl, Serializer &rdSer, WrSerializer &wrSer) { + RX_ALWAYS_INLINE bool contains([[maybe_unused]] int field) const noexcept { return false; } + RX_ALWAYS_INLINE bool match(const TagsPath &) const noexcept { return false; } + }; + + class DummyFilter { + public: + DummyFilter MakeCleanCopy() const noexcept { return DummyFilter(); } + SkipFilter MakeSkipFilter() const noexcept { return SkipFilter(); } + RX_ALWAYS_INLINE bool HasArraysFields(const PayloadTypeImpl &) const noexcept { return false; } + + RX_ALWAYS_INLINE bool contains([[maybe_unused]] int field) const noexcept { return true; } + RX_ALWAYS_INLINE bool match(const TagsPath &) const noexcept { return true; } + }; + + class IndexedSkipFilter { + public: + IndexedSkipFilter(const FieldsSet &f) noexcept : f_(&f) {} + IndexedSkipFilter MakeCleanCopy() const noexcept { return IndexedSkipFilter(*f_); } + IndexedSkipFilter MakeSkipFilter() const noexcept { return IndexedSkipFilter(*f_); } + + RX_ALWAYS_INLINE bool contains(int field) const noexcept { return f_->contains(field); } + RX_ALWAYS_INLINE bool match(const TagsPath &) const noexcept { return false; } + + private: + const FieldsSet *f_; + }; + + class RestrictingFilter { + public: + RestrictingFilter(const FieldsSet &f) noexcept : f_(&f), match_(true) {} + + RestrictingFilter MakeCleanCopy() const noexcept { return RestrictingFilter(*f_); } + IndexedSkipFilter MakeSkipFilter() const noexcept { return IndexedSkipFilter(*f_); } + RX_ALWAYS_INLINE bool HasArraysFields(const PayloadTypeImpl &pt) const noexcept { + for (auto f : *f_) { + if (f >= 0 && pt.Field(f).IsArray()) { + return true; + } + } + return false; + } + + RX_ALWAYS_INLINE bool contains(int field) noexcept { + match_ = f_->contains(field); + return match_; + } + RX_ALWAYS_INLINE bool match(const TagsPath &tagsPath) noexcept { + match_ = match_ && f_->getTagsPathsLength() && f_->match(tagsPath); + return match_; + } + + private: + const FieldsSet *f_; + bool match_; + }; + + class DummyRecoder { + public: + RX_ALWAYS_INLINE DummyRecoder MakeCleanCopy() const noexcept { return DummyRecoder(); } + RX_ALWAYS_INLINE bool Recode(Serializer &, WrSerializer &) const noexcept { return false; } + RX_ALWAYS_INLINE bool Recode(Serializer &, Payload &, [[maybe_unused]] int tagName, WrSerializer &) const noexcept { return false; } + RX_ALWAYS_INLINE TagType RegisterTagType(TagType oldTagType, [[maybe_unused]] int field) const noexcept { return oldTagType; } + RX_ALWAYS_INLINE TagType RegisterTagType(TagType oldTagType, const TagsPath &) const noexcept { return oldTagType; } + }; + class DefaultRecoder { + public: + DefaultRecoder(Recoder &r) noexcept : r_(&r), needToRecode_(false) {} + + RX_ALWAYS_INLINE DefaultRecoder MakeCleanCopy() const noexcept { return DefaultRecoder(*r_); } + + RX_ALWAYS_INLINE bool Recode(Serializer &ser, WrSerializer &wser) const { + if (needToRecode_) { + r_->Recode(ser, wser); + } + return needToRecode_; + } + RX_ALWAYS_INLINE bool Recode(Serializer &s, Payload &p, int tagName, WrSerializer &wser) const { + if (needToRecode_) { + r_->Recode(s, p, tagName, wser); + } + return needToRecode_; + } + RX_ALWAYS_INLINE TagType RegisterTagType(TagType oldTagType, int field) { + needToRecode_ = r_->Match(field); + return needToRecode_ ? r_->Type(oldTagType) : oldTagType; + } + RX_ALWAYS_INLINE TagType RegisterTagType(TagType oldTagType, const TagsPath &tagsPath) { + needToRecode_ = r_->Match(tagsPath); + return needToRecode_ ? r_->Type(oldTagType) : oldTagType; + } + + private: + Recoder *r_; + bool needToRecode_; + }; + struct NamedTagOpt {}; + struct NamelessTagOpt {}; + + template + void Decode(Payload &pl, Serializer &rdSer, WrSerializer &wrSer, FilterT filter = FilterT(), RecoderT recoder = RecoderT()) { + static_assert(std::is_same_v || std::is_same_v, + "Other filter types are not allowed for the public API"); + static_assert(std::is_same_v || std::is_same_v, + "Other recoder types are not allowed for the public API"); objectScalarIndexes_.reset(); - decodeCJson(pl, rdSer, wrSer, true); + if rx_likely (!filter.HasArraysFields(pl.Type())) { + decodeCJson(pl, rdSer, wrSer, filter, recoder, NamelessTagOpt{}); + return; + } +#ifdef RX_WITH_STDLIB_DEBUG + std::abort(); +#else + // Search of the indexed fields inside the object arrays is not imlpemented + // Possible implementation has noticable negative effect on 'FromCJSONPKOnly' benchmark. + // Currently we are using filter for PKs only, and PKs can not be arrays, so this code actually will never be called at the + // current moment + decodeCJson(pl, rdSer, wrSer, DummyFilter(), recoder, NamelessTagOpt{}); +#endif // RX_WITH_STDLIB_DEBUG } private: - bool decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrser, bool match); + template + bool decodeCJson(Payload &pl, Serializer &rdser, WrSerializer &wrser, FilterT filter, RecoderT recoder, TagOptT); bool isInArray() const noexcept { return arrayLevel_ > 0; } + [[noreturn]] void throwTagReferenceError(ctag, const Payload &); + [[noreturn]] void throwUnexpectedArrayError(const PayloadFieldType &); + + [[nodiscard]] Variant cjsonValueToVariant(TagType tag, Serializer &rdser, KeyValueType dstType); TagsMatcher &tagsMatcher_; - const FieldsSet *filter_; TagsPath tagsPath_; - Recoder *recoder_{nullptr}; int32_t arrayLevel_ = 0; ScalarIndexesSetT objectScalarIndexes_; + // storage for owning strings obtained from numbers + std::deque &storage_; }; +extern template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::DummyFilter, CJsonDecoder::DummyRecoder, CJsonDecoder::NamelessTagOpt); +extern template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::DummyFilter, CJsonDecoder::DefaultRecoder, CJsonDecoder::NamelessTagOpt); +extern template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::RestrictingFilter, CJsonDecoder::DummyRecoder, CJsonDecoder::NamelessTagOpt); +extern template bool CJsonDecoder::decodeCJson( + Payload &, Serializer &, WrSerializer &, CJsonDecoder::RestrictingFilter, CJsonDecoder::DefaultRecoder, CJsonDecoder::NamelessTagOpt); + } // namespace reindexer diff --git a/cpp_src/core/cjson/cjsontools.cc b/cpp_src/core/cjson/cjsontools.cc index 0294772ff..f7d56db0b 100644 --- a/cpp_src/core/cjson/cjsontools.cc +++ b/cpp_src/core/cjson/cjsontools.cc @@ -93,15 +93,21 @@ void copyCJsonValue(TagType tagType, Serializer &rdser, WrSerializer &wrser) { } void skipCjsonTag(ctag tag, Serializer &rdser, std::array *fieldsArrayOffsets) { - const auto field = tag.Field(); - const bool embeddedField = (field < 0); switch (tag.Type()) { case TAG_ARRAY: { + const auto field = tag.Field(); + const bool embeddedField = (field < 0); if (embeddedField) { const carraytag atag = rdser.GetCArrayTag(); - for (size_t i = 0, count = atag.Count(); i < count; ++i) { - const ctag t = atag.Type() != TAG_OBJECT ? ctag{atag.Type()} : rdser.GetCTag(); - skipCjsonTag(t, rdser); + const auto count = atag.Count(); + if (atag.Type() == TAG_OBJECT) { + for (size_t i = 0; i < count; ++i) { + skipCjsonTag(rdser.GetCTag(), rdser); + } + } else { + for (size_t i = 0; i < count; ++i) { + skipCjsonTag(ctag{atag.Type()}, rdser); + } } } else { const auto len = rdser.GetVarUint(); @@ -110,7 +116,6 @@ void skipCjsonTag(ctag tag, Serializer &rdser, std::array } } } break; - case TAG_OBJECT: for (ctag otag{rdser.GetCTag()}; otag != kCTagEnd; otag = rdser.GetCTag()) { skipCjsonTag(otag, rdser, fieldsArrayOffsets); @@ -122,12 +127,15 @@ void skipCjsonTag(ctag tag, Serializer &rdser, std::array case TAG_END: case TAG_BOOL: case TAG_NULL: - case TAG_UUID: + case TAG_UUID: { + const auto field = tag.Field(); + const bool embeddedField = (field < 0); if (embeddedField) { rdser.SkipRawVariant(KeyValueType{tag.Type()}); } else if (fieldsArrayOffsets) { (*fieldsArrayOffsets)[field] += 1; } + } } } @@ -160,4 +168,13 @@ void buildPayloadTuple(const PayloadIface &pl, const TagsMatcher *tagsMatcher template void buildPayloadTuple(const PayloadIface &, const TagsMatcher *, WrSerializer &); template void buildPayloadTuple(const PayloadIface &, const TagsMatcher *, WrSerializer &); +void throwUnexpectedNestedArrayError(std::string_view parserName, const PayloadFieldType &f) { + throw Error(errLogic, "Error parsing %s field '%s' - got value nested into the array, but expected scalar %s", parserName, f.Name(), + f.Type().Name()); +} + +void throwScalarMultipleEncodesError(const Payload &pl, const PayloadFieldType &f, int field) { + throw Error(errLogic, "Non-array field '%s' [%d] from '%s' can only be encoded once.", f.Name(), field, pl.Type().Name()); +} + } // namespace reindexer diff --git a/cpp_src/core/cjson/cjsontools.h b/cpp_src/core/cjson/cjsontools.h index 69070c1e8..553280d4e 100644 --- a/cpp_src/core/cjson/cjsontools.h +++ b/cpp_src/core/cjson/cjsontools.h @@ -17,15 +17,16 @@ void putCJsonValue(TagType tagType, int tagName, const VariantArray &values, WrS void skipCjsonTag(ctag tag, Serializer &rdser, std::array *fieldsArrayOffsets = nullptr); [[nodiscard]] Variant cjsonValueToVariant(TagType tag, Serializer &rdser, KeyValueType dstType); +[[noreturn]] void throwUnexpectedNestedArrayError(std::string_view parserName, const PayloadFieldType &f); +[[noreturn]] void throwScalarMultipleEncodesError(const Payload &pl, const PayloadFieldType &f, int field); RX_ALWAYS_INLINE void validateNonArrayFieldRestrictions(const ScalarIndexesSetT &scalarIndexes, const Payload &pl, const PayloadFieldType &f, int field, bool isInArray, std::string_view parserName) { if (!f.IsArray()) { if rx_unlikely (isInArray) { - throw Error(errLogic, "Error parsing %s field '%s' - got value nested into the array, but expected scalar %s", parserName, - f.Name(), f.Type().Name()); + throwUnexpectedNestedArrayError(parserName, f); } if rx_unlikely (scalarIndexes.test(field)) { - throw Error(errLogic, "Non-array field '%s' [%d] from '%s' can only be encoded once.", f.Name(), field, pl.Type().Name()); + throwScalarMultipleEncodesError(pl, f, field); } } } diff --git a/cpp_src/core/cjson/ctag.h b/cpp_src/core/cjson/ctag.h index f752ce006..b591ad616 100644 --- a/cpp_src/core/cjson/ctag.h +++ b/cpp_src/core/cjson/ctag.h @@ -14,7 +14,6 @@ namespace reindexer { class Serializer; class WrSerializer; -} // namespace reindexer class ctag { friend class reindexer::Serializer; @@ -52,8 +51,8 @@ class ctag { [[nodiscard]] constexpr bool operator!=(ctag other) const noexcept { return !operator==(other); } private: - explicit constexpr ctag(uint32_t tag) noexcept : ctag{typeImpl(tag), nameImpl(tag), fieldImpl(tag)} { assertrx(tag == tag_); } - explicit constexpr ctag(uint64_t tag) noexcept : ctag{typeImpl(tag), nameImpl(tag), fieldImpl(tag)} { assertrx(tag == tag_); } + explicit constexpr ctag(uint32_t tag) noexcept : ctag{typeImpl(tag), nameImpl(tag), fieldImpl(tag)} { assertrx_dbg(tag == tag_); } + explicit constexpr ctag(uint64_t tag) noexcept : ctag{typeImpl(tag), nameImpl(tag), fieldImpl(tag)} { assertrx_dbg(tag == tag_); } [[nodiscard]] constexpr static TagType typeImpl(uint32_t tag) noexcept { return static_cast((tag & kTypeMask) | ((tag >> kType1Offset) & kInvertedTypeMask)); } @@ -90,7 +89,7 @@ class carraytag { [[nodiscard]] constexpr bool operator!=(carraytag other) const noexcept { return !operator==(other); } private: - explicit constexpr carraytag(uint32_t atag) noexcept : carraytag{countImpl(atag), typeImpl(atag)} { assertrx(atag == atag_); } + explicit constexpr carraytag(uint32_t atag) noexcept : carraytag{countImpl(atag), typeImpl(atag)} { assertrx_dbg(atag == atag_); } [[nodiscard]] constexpr uint32_t asNumber() const noexcept { return atag_; } [[nodiscard]] static constexpr TagType typeImpl(uint32_t atag) noexcept { return static_cast((atag >> kCountBits) & kTypeMask); @@ -99,3 +98,5 @@ class carraytag { uint32_t atag_; }; + +} // namespace reindexer diff --git a/cpp_src/core/cjson/fieldextractor.h b/cpp_src/core/cjson/fieldextractor.h index 6b11f179a..43ac72d64 100644 --- a/cpp_src/core/cjson/fieldextractor.h +++ b/cpp_src/core/cjson/fieldextractor.h @@ -16,7 +16,7 @@ class FieldsExtractor { }; FieldsExtractor() = default; - FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, FieldsSet *filter = nullptr, + FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, const FieldsSet *filter, FieldParams *params = nullptr) noexcept : values_(va), expectedType_(expectedType), expectedPathDepth_(expectedPathDepth), filter_(filter), params_(params) {} FieldsExtractor(FieldsExtractor &&other) = default; @@ -176,7 +176,7 @@ class FieldsExtractor { VariantArray *values_ = nullptr; KeyValueType expectedType_{KeyValueType::Undefined{}}; int expectedPathDepth_ = 0; - FieldsSet *filter_; + const FieldsSet *filter_; FieldParams *params_; }; diff --git a/cpp_src/core/cjson/tagsmatcher.h b/cpp_src/core/cjson/tagsmatcher.h index 8bb8786b0..57fb2e5e8 100644 --- a/cpp_src/core/cjson/tagsmatcher.h +++ b/cpp_src/core/cjson/tagsmatcher.h @@ -21,12 +21,12 @@ class TagsMatcher { int res = impl_->name2tag(name); return res ? res : impl_.clone()->name2tag(name, canAdd, updated_); } - int tags2field(const int16_t* path, size_t pathLen) const { return impl_->tags2field(path, pathLen); } + int tags2field(const int16_t* path, size_t pathLen) const noexcept { return impl_->tags2field(path, pathLen); } const std::string& tag2name(int tag) const { return impl_->tag2name(tag); } TagsPath path2tag(std::string_view jsonPath) const { return impl_->path2tag(jsonPath); } TagsPath path2tag(std::string_view jsonPath, bool canAdd) { - auto res = path2tag(jsonPath); if (jsonPath.empty()) return TagsPath(); + auto res = path2tag(jsonPath); return res.empty() && canAdd ? impl_.clone()->path2tag(jsonPath, canAdd, updated_) : res; } IndexedTagsPath path2indexedtag(std::string_view jsonPath, const IndexExpressionEvaluator& ev) const { diff --git a/cpp_src/core/cjson/tagsmatcherimpl.h b/cpp_src/core/cjson/tagsmatcherimpl.h index e96109868..80577249c 100644 --- a/cpp_src/core/cjson/tagsmatcherimpl.h +++ b/cpp_src/core/cjson/tagsmatcherimpl.h @@ -4,12 +4,12 @@ #include #include -#include "core/keyvalue/key_string.h" #include "core/payload/payloadtype.h" #include "core/payload/payloadtypeimpl.h" #include "ctag.h" #include "tagspath.h" #include "tagspathcache.h" +#include "tools/randomgenerator.h" #include "tools/serializer.h" #include "tools/stringstools.h" @@ -17,9 +17,9 @@ namespace reindexer { class TagsMatcherImpl { public: - TagsMatcherImpl() : version_(0), stateToken_(rand()) {} - TagsMatcherImpl(PayloadType payloadType) : payloadType_(std::move(payloadType)), version_(0), stateToken_(rand()) {} - ~TagsMatcherImpl() {} + TagsMatcherImpl() : version_(0), stateToken_(tools::RandomGenerator::gets32()) {} + TagsMatcherImpl(PayloadType &&payloadType) + : payloadType_(std::move(payloadType)), version_(0), stateToken_(tools::RandomGenerator::gets32()) {} TagsPath path2tag(std::string_view jsonPath) const { bool updated = false; @@ -75,25 +75,29 @@ class TagsMatcherImpl { if (content == "*"sv) { node.MarkAllItems(true); } else { - int index = stoi(content); - if (index == 0 && content != "0"sv && ev) { - VariantArray values = ev(content); - if (values.size() != 1) { - throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content); + auto index = try_stoi(content); + if (!index) { + if (ev) { + VariantArray values = ev(content); + if (values.size() != 1) { + throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content); + } + values.front().Type().EvaluateOneOf( + [](OneOf) noexcept {}, + [&](OneOf) { + throw Error(errParams, "Wrong type of index: '%s'", content); + }); + node.SetExpression(content); + index = values.front().As(); + } else { + throw Error(errParams, "Can't convert '%s' to number", content); } - values.front().Type().EvaluateOneOf( - [](OneOf) noexcept {}, - [&](OneOf) { - throw Error(errParams, "Wrong type of index: '%s'", content); - }); - node.SetExpression(content); - index = values.front().As(); } if (index < 0) { throw Error(errLogic, "Array index value cannot be negative"); } - node.SetIndex(index); + node.SetIndex(*index); } field = field.substr(0, openBracketPos); } @@ -140,7 +144,7 @@ class TagsMatcherImpl { return tags2names_[tag - 1]; } - int tags2field(const int16_t *path, size_t pathLen) const { + int tags2field(const int16_t *path, size_t pathLen) const noexcept { if (!pathLen) return -1; return pathCache_.lookup(path, pathLen); } diff --git a/cpp_src/core/cjson/tagspathcache.h b/cpp_src/core/cjson/tagspathcache.h index fecd8d3de..26e0240f9 100644 --- a/cpp_src/core/cjson/tagspathcache.h +++ b/cpp_src/core/cjson/tagspathcache.h @@ -31,7 +31,7 @@ class TagsPathCache { len--; } } - int lookup(const int16_t *tagsPath, size_t len) const { + int lookup(const int16_t *tagsPath, size_t len) const noexcept { assertrx(len); auto cache = this; for (;;) { @@ -51,7 +51,7 @@ class TagsPathCache { } } - void walk(int16_t *path, int depth, const std::function& visitor) const { + void walk(int16_t *path, int depth, const std::function &visitor) const { int16_t &i = path[depth]; for (i = 0; i < int(entries_.size()); i++) { if (entries_[i].field_ > 0) visitor(depth + 1, entries_[i].field_); diff --git a/cpp_src/core/comparator.cc b/cpp_src/core/comparator.cc index 08d001339..c105f53f2 100644 --- a/cpp_src/core/comparator.cc +++ b/cpp_src/core/comparator.cc @@ -82,16 +82,11 @@ void Comparator::Bind(const PayloadType &type, int field) { } } -void Comparator::BindEqualPosition(int field, const VariantArray &val, CondType cond) { cmpEqualPosition.BindField(field, val, cond); } - -void Comparator::BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond) { - cmpEqualPosition.BindField(tagsPath, val, cond); -} - bool Comparator::isNumericComparison(const VariantArray &values) const { if (valuesType_.Is() || values.empty()) return false; const KeyValueType keyType{values.front().Type()}; - return !valuesType_.IsSame(keyType) && (valuesType_.Is() || keyType.Is()); + return !valuesType_.IsSame(keyType) && ((valuesType_.Is() && !keyType.Is()) || + (keyType.Is() && !valuesType_.Is())); } bool Comparator::Compare(const PayloadValue &data, int rowId) { @@ -191,7 +186,7 @@ void Comparator::ExcludeDistinct(const PayloadValue &data, int rowId) { } else { // Exclude field from payload by offset (fast path) - assertrx(!type_.Is()); + assertrx_throw(!type_.Is()); // Check if we have column (rawData_), then go to fastest path with column if (rawData_) return excludeDistinct(rawData_ + rowId * sizeof_); diff --git a/cpp_src/core/comparator.h b/cpp_src/core/comparator.h index 3760077c8..7b1a05445 100644 --- a/cpp_src/core/comparator.h +++ b/cpp_src/core/comparator.h @@ -15,8 +15,10 @@ class Comparator : public ComparatorVars { bool Compare(const PayloadValue &lhs, int rowId); void ExcludeDistinct(const PayloadValue &, int rowId); void Bind(const PayloadType &type, int field); - void BindEqualPosition(int field, const VariantArray &val, CondType cond); - void BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond); + template + void BindEqualPosition(F &&field, const VariantArray &val, CondType cond) { + cmpEqualPosition.BindField(std::forward(field), val, cond); + } void ClearDistinct() { cmpInt.ClearDistinct(); cmpBool.ClearDistinct(); @@ -116,7 +118,7 @@ class Comparator : public ComparatorVars { ComparatorImpl cmpComposite; ComparatorImpl cmpGeom; ComparatorImpl cmpUuid; - CompositeArrayComparator cmpEqualPosition; + EqualPositionComparator cmpEqualPosition; KeyValueType valuesType_{KeyValueType::Undefined{}}; }; diff --git a/cpp_src/core/comparatorimpl.h b/cpp_src/core/comparatorimpl.h index 41f0aa486..5da4466bc 100644 --- a/cpp_src/core/comparatorimpl.h +++ b/cpp_src/core/comparatorimpl.h @@ -349,10 +349,12 @@ class ComparatorImpl { void SetValues(CondType cond, const VariantArray &values, const ComparatorVars &vars) { if (cond == CondSet) { valuesSet_.reset(new intrusive_atomic_rc_wrapper( - values.size(), hash_composite(vars.payloadType_, vars.fields_), equal_composite(vars.payloadType_, vars.fields_))); + values.size(), hash_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}), + equal_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}))); } else if (cond == CondAllSet) { valuesSet_.reset(new intrusive_atomic_rc_wrapper( - values.size(), hash_composite(vars.payloadType_, vars.fields_), equal_composite(vars.payloadType_, vars.fields_))); + values.size(), hash_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}), + equal_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}))); allSetValuesSet_.reset(new intrusive_atomic_rc_wrapper>{}); } diff --git a/cpp_src/core/compositearraycomparator.cc b/cpp_src/core/compositearraycomparator.cc index 6e58fdf98..4ffb48e83 100644 --- a/cpp_src/core/compositearraycomparator.cc +++ b/cpp_src/core/compositearraycomparator.cc @@ -2,37 +2,29 @@ #include namespace reindexer { -CompositeArrayComparator::CompositeArrayComparator() {} +void EqualPositionComparator::BindField(int field, const VariantArray &values, CondType cond) { bindField(field, values, cond); } -void CompositeArrayComparator::BindField(int field, const VariantArray &values, CondType condType) { +void EqualPositionComparator::BindField(const FieldsPath &fieldPath, const VariantArray &values, CondType cond) { + bindField(fieldPath, values, cond); +} + +template +void EqualPositionComparator::bindField(F field, const VariantArray &values, CondType cond) { fields_.push_back(field); Context &ctx = ctx_.emplace_back(); - ctx.cond = condType; - ctx.cmpBool.SetValues(condType, values); - ctx.cmpInt.SetValues(condType, values); - ctx.cmpInt64.SetValues(condType, values); - ctx.cmpString.SetValues(condType, values, CollateOpts()); - ctx.cmpDouble.SetValues(condType, values); - ctx.cmpUuid.SetValues(condType, values); + ctx.cond = cond; + ctx.cmpBool.SetValues(cond, values); + ctx.cmpInt.SetValues(cond, values); + ctx.cmpInt64.SetValues(cond, values); + ctx.cmpString.SetValues(cond, values, CollateOpts()); + ctx.cmpDouble.SetValues(cond, values); + ctx.cmpUuid.SetValues(cond, values); assertrx(ctx_.size() == fields_.size()); } -void CompositeArrayComparator::BindField(const TagsPath &tagsPath, const VariantArray &values, CondType condType) { - fields_.push_back(tagsPath); - Context &ctx = ctx_.emplace_back(); - - ctx.cond = condType; - ctx.cmpBool.SetValues(condType, values); - ctx.cmpInt.SetValues(condType, values); - ctx.cmpInt64.SetValues(condType, values); - ctx.cmpString.SetValues(condType, values, CollateOpts()); - ctx.cmpDouble.SetValues(condType, values); - ctx.cmpUuid.SetValues(condType, values); -} - -bool CompositeArrayComparator::Compare(const PayloadValue &pv, const ComparatorVars &vars) { +bool EqualPositionComparator::Compare(const PayloadValue &pv, const ComparatorVars &vars) { ConstPayload pl(vars.payloadType_, pv); size_t len = INT_MAX; @@ -64,7 +56,7 @@ bool CompositeArrayComparator::Compare(const PayloadValue &pv, const ComparatorV return false; } -bool CompositeArrayComparator::compareField(size_t field, const Variant &v, const ComparatorVars &vars) { +bool EqualPositionComparator::compareField(size_t field, const Variant &v, const ComparatorVars &vars) { return v.Type().EvaluateOneOf( [&](KeyValueType::Bool) { return ctx_[field].cmpBool.Compare(ctx_[field].cond, static_cast(v)); }, [&](KeyValueType::Int) { return ctx_[field].cmpInt.Compare(ctx_[field].cond, static_cast(v)); }, diff --git a/cpp_src/core/compositearraycomparator.h b/cpp_src/core/compositearraycomparator.h index d646ed6b4..a58735dc6 100644 --- a/cpp_src/core/compositearraycomparator.h +++ b/cpp_src/core/compositearraycomparator.h @@ -5,17 +5,19 @@ namespace reindexer { -class CompositeArrayComparator { +class EqualPositionComparator { public: - CompositeArrayComparator(); + EqualPositionComparator() noexcept = default; - void BindField(int field, const VariantArray &values, CondType condType); - void BindField(const TagsPath &tagsPath, const VariantArray &values, CondType condType); - bool Compare(const PayloadValue &pv, const ComparatorVars &vars); + void BindField(int field, const VariantArray &, CondType); + void BindField(const FieldsPath &, const VariantArray &, CondType); + bool Compare(const PayloadValue &, const ComparatorVars &); bool IsBinded() { return !ctx_.empty(); } private: - bool compareField(size_t field, const Variant &v, const ComparatorVars &vars); + bool compareField(size_t field, const Variant &, const ComparatorVars &); + template + void bindField(F field, const VariantArray &, CondType); struct Context { CondType cond; diff --git a/cpp_src/core/dbconfig.cc b/cpp_src/core/dbconfig.cc index d2b4aa462..60ebb6550 100644 --- a/cpp_src/core/dbconfig.cc +++ b/cpp_src/core/dbconfig.cc @@ -45,8 +45,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { LongTxLoggingParams{profilingNode["long_queries_logging"]["transaction"]["threshold_us"].As(), profilingNode["long_queries_logging"]["transaction"]["avg_step_threshold_us"].As()}); } - auto it = handlers_.find(ProfilingConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[ProfilingConf]) (handlers_[ProfilingConf])(); } auto &namespacesNode = root["namespaces"]; @@ -73,10 +72,29 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { data.maxPreselectPart = nsNode["max_preselect_part"].As(data.maxPreselectPart, 0.0, 1.0); data.idxUpdatesCountingMode = nsNode["index_updates_counting_mode"].As(data.idxUpdatesCountingMode); data.syncStorageFlushLimit = nsNode["sync_storage_flush_limit"].As(data.syncStorageFlushLimit, 0); + + auto cacheConfig = nsNode["cache"]; + if (!cacheConfig.empty()) { + data.cacheConfig.idxIdsetCacheSize = + cacheConfig["index_idset_cache_size"].As(data.cacheConfig.idxIdsetCacheSize, 0); + data.cacheConfig.idxIdsetHitsToCache = + cacheConfig["index_idset_hits_to_cache"].As(data.cacheConfig.idxIdsetHitsToCache, 0); + data.cacheConfig.ftIdxCacheSize = cacheConfig["ft_index_cache_size"].As(data.cacheConfig.ftIdxCacheSize, 0); + data.cacheConfig.ftIdxHitsToCache = + cacheConfig["ft_index_hits_to_cache"].As(data.cacheConfig.ftIdxHitsToCache, 0); + data.cacheConfig.joinCacheSize = + cacheConfig["joins_preselect_cache_size"].As(data.cacheConfig.joinCacheSize, 0); + data.cacheConfig.joinHitsToCache = + cacheConfig["joins_preselect_hit_to_cache"].As(data.cacheConfig.joinHitsToCache, 0); + data.cacheConfig.queryCountCacheSize = + cacheConfig["query_count_cache_size"].As(data.cacheConfig.queryCountCacheSize, 0); + data.cacheConfig.queryCountHitsToCache = + cacheConfig["query_count_hit_to_cache"].As(data.cacheConfig.queryCountHitsToCache, 0); + } + namespacesData_.emplace(nsNode["namespace"].As(), std::move(data)); // NOLINT(performance-move-const-arg) } - auto it = handlers_.find(NamespaceDataConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[NamespaceDataConf]) (handlers_[NamespaceDataConf])(); } auto &replicationNode = root["replication"]; @@ -84,8 +102,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { auto err = replicationData_.FromJSON(replicationNode); if (!err.ok()) return err; - auto it = handlers_.find(ReplicationConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[ReplicationConf]) (handlers_[ReplicationConf])(); } return errOK; } catch (const Error &err) { @@ -105,11 +122,11 @@ ReplicationConfigData DBConfigProvider::GetReplicationConfig() { return replicationData_; } -bool DBConfigProvider::GetNamespaceConfig(const std::string &nsName, NamespaceConfigData &data) { +bool DBConfigProvider::GetNamespaceConfig(std::string_view nsName, NamespaceConfigData &data) { shared_lock lk(mtx_); auto it = namespacesData_.find(nsName); if (it == namespacesData_.end()) { - it = namespacesData_.find("*"); + it = namespacesData_.find(std::string_view("*")); } if (it == namespacesData_.end()) { data = {}; diff --git a/cpp_src/core/dbconfig.h b/cpp_src/core/dbconfig.h index cbb4b1bed..fca5bae6d 100644 --- a/cpp_src/core/dbconfig.h +++ b/cpp_src/core/dbconfig.h @@ -2,7 +2,7 @@ #include #include -#include +#include "estl/fast_hash_map.h" #include "estl/fast_hash_set.h" #include "estl/mutex.h" #include "estl/shared_mutex.h" @@ -18,7 +18,13 @@ class JsonBuilder; class RdxContext; class WrSerializer; -enum ConfigType { ProfilingConf, NamespaceDataConf, ReplicationConf }; +enum ConfigType { + ProfilingConf = 0, + NamespaceDataConf, + ReplicationConf, + // + kConfigTypesTotalCount +}; class LongQueriesLoggingParams { public: @@ -65,6 +71,31 @@ class ProfilingConfigData { std::atomic longTxLoggingParams; }; +constexpr size_t kDefaultCacheSizeLimit = 1024 * 1024 * 128; +constexpr uint32_t kDefaultHitCountToCache = 2; + +struct NamespaceCacheConfigData { + bool IsIndexesCacheEqual(const NamespaceCacheConfigData &o) noexcept { + return idxIdsetCacheSize == o.idxIdsetCacheSize && idxIdsetHitsToCache == o.idxIdsetHitsToCache && + ftIdxCacheSize == o.ftIdxCacheSize && ftIdxHitsToCache == o.ftIdxHitsToCache; + } + bool IsJoinCacheEqual(const NamespaceCacheConfigData &o) noexcept { + return joinCacheSize == o.joinCacheSize && joinHitsToCache == o.joinHitsToCache; + } + bool IsQueryCountCacheEqual(const NamespaceCacheConfigData &o) noexcept { + return queryCountCacheSize == o.queryCountCacheSize && queryCountHitsToCache == o.queryCountHitsToCache; + } + + uint64_t idxIdsetCacheSize = kDefaultCacheSizeLimit; + uint32_t idxIdsetHitsToCache = kDefaultHitCountToCache; + uint64_t ftIdxCacheSize = kDefaultCacheSizeLimit; + uint32_t ftIdxHitsToCache = kDefaultHitCountToCache; + uint64_t joinCacheSize = 2 * kDefaultCacheSizeLimit; + uint32_t joinHitsToCache = kDefaultHitCountToCache; + uint64_t queryCountCacheSize = kDefaultCacheSizeLimit; + uint32_t queryCountHitsToCache = kDefaultHitCountToCache; +}; + struct NamespaceConfigData { bool lazyLoad = false; int noQueryIdleThreshold = 0; @@ -82,6 +113,7 @@ struct NamespaceConfigData { double maxPreselectPart = 0.1; bool idxUpdatesCountingMode = false; int syncStorageFlushLimit = 20000; + NamespaceCacheConfigData cacheConfig; }; enum ReplicationRole { ReplicationNone, ReplicationMaster, ReplicationSlave, ReplicationReadOnly }; @@ -133,7 +165,7 @@ class DBConfigProvider { void setHandler(ConfigType cfgType, std::function handler); ReplicationConfigData GetReplicationConfig(); - bool GetNamespaceConfig(const std::string &nsName, NamespaceConfigData &data); + bool GetNamespaceConfig(std::string_view nsName, NamespaceConfigData &data); LongQueriesLoggingParams GetSelectLoggingParams() const noexcept { return profilingData_.longSelectLoggingParams.load(std::memory_order_relaxed); } @@ -150,8 +182,8 @@ class DBConfigProvider { private: ProfilingConfigData profilingData_; ReplicationConfigData replicationData_; - std::unordered_map namespacesData_; - std::unordered_map> handlers_; + fast_hash_map namespacesData_; + std::array, kConfigTypesTotalCount> handlers_; shared_timed_mutex mtx_; }; diff --git a/cpp_src/core/defnsconfigs.h b/cpp_src/core/defnsconfigs.h index 36bcd02b4..04fec8aa3 100644 --- a/cpp_src/core/defnsconfigs.h +++ b/cpp_src/core/defnsconfigs.h @@ -56,7 +56,17 @@ const std::vector kDefDBConfig = { "max_preselect_size":1000, "max_preselect_part":0.1, "index_updates_counting_mode":false, - "sync_storage_flush_limit":20000 + "sync_storage_flush_limit":20000, + "cache":{ + "index_idset_cache_size":134217728, + "index_idset_hits_to_cache":2, + "ft_index_cache_size":134217728, + "ft_index_hits_to_cache":2, + "joins_preselect_cache_size":268435456, + "joins_preselect_hit_to_cache":2, + "query_count_cache_size":134217728, + "query_count_hit_to_cache":2 + } } ] })json", @@ -114,7 +124,6 @@ const std::vector kSystemNsDefs = { .AddIndex("last_sec_avg_lock_time_us", "-", "int64", IndexOpts().Dense()) .AddIndex("latency_stddev", "-", "double", IndexOpts().Dense()), NamespaceDef(kNamespacesNamespace, StorageOpts()).AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()), - NamespaceDef(kPerfStatsNamespace, StorageOpts()).AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()), NamespaceDef(kMemStatsNamespace, StorageOpts()) .AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()) .AddIndex("items_count", "-", "int64", IndexOpts().Dense()) diff --git a/cpp_src/core/expressiontree.h b/cpp_src/core/expressiontree.h index 6f4147452..0eeaced61 100644 --- a/cpp_src/core/expressiontree.h +++ b/cpp_src/core/expressiontree.h @@ -283,9 +283,10 @@ class ExpressionTree { }; public: - Node() : storage_{SubTree{1}} {} + Node() : storage_{std::in_place_type, 1} {} template - Node(OperationType op, size_t s, Args&&... args) : storage_{SubTree{s, std::forward(args)...}}, operation{op} {} + Node(OperationType op, size_t s, Args&&... args) + : storage_{std::in_place_type, s, std::forward(args)...}, operation{op} {} template Node(OperationType op, T&& v) : storage_{std::forward(v)}, operation{op} {} Node(const Node& other) : storage_{other.storage_}, operation{other.operation} {} @@ -363,6 +364,10 @@ class ExpressionTree { void SetValue(T&& v) { storage_ = std::forward(v); } + template + void Emplace(Args&&... args) { + storage_.template emplace(std::forward(args)...); + } private: Storage storage_; @@ -443,6 +448,15 @@ class ExpressionTree { } container_.emplace_back(op, v); } + /// Appends value to the last openned subtree + template + void Append(OperationType op, Args&&... args) { + for (unsigned i : activeBrackets_) { + assertrx(i < container_.size()); + container_[i].Append(); + } + container_.emplace_back(op, T{std::forward(args)...}); + } class const_iterator; /// Appends all nodes from the interval to the last openned subtree void Append(const_iterator begin, const_iterator end) { diff --git a/cpp_src/core/ft/ftsetcashe.h b/cpp_src/core/ft/ftsetcashe.h index 810a06ccd..95e64f014 100644 --- a/cpp_src/core/ft/ftsetcashe.h +++ b/cpp_src/core/ft/ftsetcashe.h @@ -17,6 +17,6 @@ struct FtIdSetCacheVal { FtCtx::Data::Ptr ctx; }; -class FtIdSetCache : public LRUCache {}; +using FtIdSetCache = LRUCache; } // namespace reindexer diff --git a/cpp_src/core/iclientsstats.cc b/cpp_src/core/iclientsstats.cc index c244d5b33..2e52eb5d1 100644 --- a/cpp_src/core/iclientsstats.cc +++ b/cpp_src/core/iclientsstats.cc @@ -8,6 +8,7 @@ void ClientStat::GetJSON(WrSerializer& ser) const { JsonBuilder builder(ser); builder.Put("connection_id", connectionId); builder.Put("ip", ip); + builder.Put("protocol", protocol); builder.Put("user_name", userName); builder.Put("db_name", dbName); builder.Put("current_activity", currentActivity); diff --git a/cpp_src/core/iclientsstats.h b/cpp_src/core/iclientsstats.h index a19f1989f..5c319034a 100644 --- a/cpp_src/core/iclientsstats.h +++ b/cpp_src/core/iclientsstats.h @@ -7,11 +7,15 @@ namespace reindexer { +constexpr std::string_view kTcpProtocolName = "tcp"; +constexpr std::string_view kUnixProtocolName = "unix"; + class WrSerializer; struct ClientStat { void GetJSON(WrSerializer& ser) const; int connectionId = 0; + std::string_view protocol = kTcpProtocolName; std::string ip; std::string userName; std::string dbName; @@ -43,6 +47,7 @@ struct ClientConnectionStat { std::shared_ptr connectionStat; std::shared_ptr txStats; std::string ip; + std::string_view protocol = kTcpProtocolName; std::string userName; std::string dbName; std::string userRights; diff --git a/cpp_src/core/idset.cc b/cpp_src/core/idset.cc index c8576fc94..da7047de7 100644 --- a/cpp_src/core/idset.cc +++ b/cpp_src/core/idset.cc @@ -1,18 +1,8 @@ #include "core/idset.h" -#include #include "tools/errors.h" namespace reindexer { -void IdSet::Commit() { - if (!size() && set_) { - resize(0); - for (auto id : *set_) push_back(id); - } - - usingBtree_.store(false, std::memory_order_release); -} - std::string IdSetPlain::Dump() const { std::string buf = "["; diff --git a/cpp_src/core/idset.h b/cpp_src/core/idset.h index 6c193f9c4..81522fcef 100644 --- a/cpp_src/core/idset.h +++ b/cpp_src/core/idset.h @@ -74,7 +74,7 @@ class IdSetPlain : protected base_idset { std::string Dump() const; protected: - IdSetPlain(base_idset &&idset) : base_idset(std::move(idset)) {} + IdSetPlain(base_idset &&idset) noexcept : base_idset(std::move(idset)) {} }; std::ostream &operator<<(std::ostream &, const IdSetPlain &); @@ -87,7 +87,7 @@ class IdSet : public IdSetPlain { public: using Ptr = intrusive_ptr>; - IdSet() : usingBtree_(false) {} + IdSet() noexcept : usingBtree_(false) {} IdSet(const IdSet &other) : IdSetPlain(other), set_(!other.set_ ? nullptr : new base_idsetset(*other.set_)), usingBtree_(other.usingBtree_.load()) {} IdSet(IdSet &&other) noexcept : IdSetPlain(std::move(other)), set_(std::move(other.set_)), usingBtree_(other.usingBtree_.load()) {} @@ -194,18 +194,24 @@ class IdSet : public IdSetPlain { base_idset::erase(d.first, d.second); return d.second - d.first; } else { - resize(0); + clear(); usingBtree_.store(true, std::memory_order_release); return set_->erase(id); } - return 0; } - void Commit(); - bool IsCommited() const { return !usingBtree_.load(std::memory_order_acquire); } - bool IsEmpty() const { return empty() && (!set_ || set_->empty()); } - size_t Size() const { return usingBtree_.load(std::memory_order_acquire) ? set_->size() : size(); } - size_t BTreeSize() const { return set_ ? sizeof(*set_.get()) + set_->size() * sizeof(int) : 0; } - const base_idsetset *BTree() const { return set_.get(); } + void Commit() { + if (!size() && set_) { + reserve(set_->size()); + for (auto id : *set_) push_back(id); + } + + usingBtree_.store(false, std::memory_order_release); + } + bool IsCommited() const noexcept { return !usingBtree_.load(std::memory_order_acquire); } + bool IsEmpty() const noexcept { return empty() && (!set_ || set_->empty()); } + size_t Size() const noexcept { return usingBtree_.load(std::memory_order_acquire) ? set_->size() : size(); } + size_t BTreeSize() const noexcept { return set_ ? sizeof(*set_.get()) + set_->size() * sizeof(int) : 0; } + const base_idsetset *BTree() const noexcept { return set_.get(); } void ReserveForSorted(int sortedIdxCount) { reserve(((set_ ? set_->size() : size())) * (sortedIdxCount + 1)); } protected: @@ -214,7 +220,7 @@ class IdSet : public IdSetPlain { template friend class BtreeIndexReverseIteratorImpl; - IdSet(base_idset &&idset) : IdSetPlain(std::move(idset)), usingBtree_(false) {} + IdSet(base_idset &&idset) noexcept : IdSetPlain(std::move(idset)), usingBtree_(false) {} std::unique_ptr set_; std::atomic usingBtree_; diff --git a/cpp_src/core/idsetcache.h b/cpp_src/core/idsetcache.h index 3c2fd9184..185073c22 100644 --- a/cpp_src/core/idsetcache.h +++ b/cpp_src/core/idsetcache.h @@ -83,8 +83,11 @@ struct hash_idset_cache_key { size_t operator()(const IdSetCacheKey &s) const { return (s.cond << 8) ^ (s.sort << 16) ^ s.keys->Hash(); } }; -class IdSetCache : public LRUCache { +using IdSetCacheBase = LRUCache; + +class IdSetCache : public IdSetCacheBase { public: + IdSetCache(size_t sizeLimit, uint32_t hitCount) : IdSetCacheBase(sizeLimit, hitCount) {} void ClearSorted(const std::bitset &s) { if (s.any()) { Clear([&s](const IdSetCacheKey &k) { return s.test(k.sort); }); diff --git a/cpp_src/core/index/index.cc b/cpp_src/core/index/index.cc index 552033fc2..cd3ba6d3c 100644 --- a/cpp_src/core/index/index.cc +++ b/cpp_src/core/index/index.cc @@ -1,5 +1,4 @@ #include "index.h" -#include "core/namespacedef.h" #include "indexordered.h" #include "indextext/fastindextext.h" #include "indextext/fuzzyindextext.h" @@ -10,8 +9,8 @@ namespace reindexer { -Index::Index(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) - : type_(idef.Type()), name_(idef.name_), opts_(idef.opts_), payloadType_(std::move(payloadType)), fields_(fields) { +Index::Index(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) + : type_(idef.Type()), name_(idef.name_), opts_(idef.opts_), payloadType_(std::move(payloadType)), fields_(std::move(fields)) { logPrintf(LogTrace, "Index::Index ('%s',%s,%s) %s%s%s", idef.name_, idef.indexType_, idef.fieldType_, idef.opts_.IsPK() ? ",pk" : "", idef.opts_.IsDense() ? ",dense" : "", idef.opts_.IsArray() ? ",array" : ""); } @@ -28,38 +27,39 @@ Index::Index(const Index& obj) selectKeyType_(obj.selectKeyType_), sortedIdxCount_(obj.sortedIdxCount_) {} -std::unique_ptr Index::New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) { +std::unique_ptr Index::New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, + const NamespaceCacheConfigData& cacheCfg) { switch (idef.Type()) { case IndexStrBTree: case IndexIntBTree: case IndexDoubleBTree: case IndexInt64BTree: case IndexCompositeBTree: - return IndexOrdered_New(idef, std::move(payloadType), fields); + return IndexOrdered_New(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexStrHash: case IndexIntHash: case IndexInt64Hash: case IndexCompositeHash: - return IndexUnordered_New(idef, std::move(payloadType), fields); + return IndexUnordered_New(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexIntStore: case IndexStrStore: case IndexInt64Store: case IndexDoubleStore: case IndexBool: case IndexUuidStore: - return IndexStore_New(idef, std::move(payloadType), fields); + return IndexStore_New(idef, std::move(payloadType), std::move(fields)); case IndexFastFT: case IndexCompositeFastFT: - return FastIndexText_New(idef, std::move(payloadType), fields); + return FastIndexText_New(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexFuzzyFT: case IndexCompositeFuzzyFT: - return FuzzyIndexText_New(idef, std::move(payloadType), fields); + return FuzzyIndexText_New(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexTtl: - return TtlIndex_New(idef, std::move(payloadType), fields); + return TtlIndex_New(idef, std::move(payloadType), std::move(fields), cacheCfg); case ::IndexRTree: - return IndexRTree_New(idef, std::move(payloadType), fields); + return IndexRTree_New(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexUuidHash: - return IndexUuid_New(idef, std::move(payloadType), fields); + return IndexUuid_New(idef, std::move(payloadType), std::move(fields), cacheCfg); } throw Error(errParams, "Ivalid index type %d for index '%s'", idef.Type(), idef.name_); } diff --git a/cpp_src/core/index/index.h b/cpp_src/core/index/index.h index 136a97fdf..0e4fe3ccf 100644 --- a/cpp_src/core/index/index.h +++ b/cpp_src/core/index/index.h @@ -12,7 +12,6 @@ #include "core/payload/payloadiface.h" #include "core/perfstatcounter.h" #include "core/selectkeyresult.h" -#include "core/type_consts_helpers.h" #include "ft_preselect.h" #include "indexiterator.h" @@ -46,7 +45,7 @@ class Index { using KeyEntry = reindexer::KeyEntry; using KeyEntryPlain = reindexer::KeyEntry; - Index(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); + Index(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields); Index(const Index&); Index& operator=(const Index&) = delete; virtual ~Index() = default; @@ -82,20 +81,24 @@ class Index { virtual bool IsDestroyPartSupported() const noexcept { return false; } virtual void AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue&) {} - const PayloadType& GetPayloadType() const { return payloadType_; } - void UpdatePayloadType(PayloadType payloadType) { payloadType_ = std::move(payloadType); } + const PayloadType& GetPayloadType() const& { return payloadType_; } + const PayloadType& GetPayloadType() const&& = delete; + void UpdatePayloadType(PayloadType&& payloadType) { payloadType_ = std::move(payloadType); } - static std::unique_ptr New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields_); + static std::unique_ptr New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields_, + const NamespaceCacheConfigData& cacheCfg); KeyValueType KeyType() const { return keyType_; } KeyValueType SelectKeyType() const { return selectKeyType_; } - const FieldsSet& Fields() const { return fields_; } - const std::string& Name() const { return name_; } + const FieldsSet& Fields() const& noexcept { return fields_; } + const FieldsSet& Fields() const&& = delete; + const std::string& Name() const& noexcept { return name_; } + const std::string& Name() const&& = delete; IndexType Type() const { return type_; } const std::vector& SortOrders() const { return sortOrders_; } const IndexOpts& Opts() const { return opts_; } virtual void SetOpts(const IndexOpts& opts) { opts_ = opts; } - virtual void SetFields(FieldsSet&& fields) { fields_ = std::move(fields); } + void SetFields(FieldsSet&& fields) { fields_ = std::move(fields); } [[nodiscard]] SortType SortId() const noexcept { return sortId_; } virtual void SetSortedIdxCount(int sortedIdxCount) { sortedIdxCount_ = sortedIdxCount; } virtual FtMergeStatuses GetFtMergeStatuses(const RdxContext&) { @@ -124,6 +127,7 @@ class Index { virtual bool IsBuilt() const noexcept { return isBuilt_; } virtual void MarkBuilt() noexcept { isBuilt_ = true; } virtual void EnableUpdatesCountingMode(bool) noexcept {} + virtual void ReconfigureCache(const NamespaceCacheConfigData& cacheCfg) = 0; virtual void Dump(std::ostream& os, std::string_view step = " ", std::string_view offset = "") const { dump(os, step, offset); } @@ -140,8 +144,12 @@ class Index { IndexOpts opts_; // Payload type of items mutable PayloadType payloadType_; - // Fields in index. Valid only for composite indexes + +private: + // Fields in index FieldsSet fields_; + +protected: // Perfstat counter PerfStatCounterMT commitPerfCounter_; PerfStatCounterMT selectPerfCounter_; diff --git a/cpp_src/core/index/indexordered.cc b/cpp_src/core/index/indexordered.cc index 828030b1b..643562e35 100644 --- a/cpp_src/core/index/indexordered.cc +++ b/cpp_src/core/index/indexordered.cc @@ -11,7 +11,7 @@ template Variant IndexOrdered::Upsert(const Variant &key, IdType id, bool &clearCache) { if (key.Type().Is()) { if (this->empty_ids_.Unsorted().Add(id, IdSet::Auto, this->sortedIdxCount_)) { - if (this->cache_) this->cache_.reset(); + this->cache_.reset(); clearCache = true; this->isBuilt_ = false; } @@ -28,7 +28,7 @@ Variant IndexOrdered::Upsert(const Variant &key, IdType id, bool &clearCache) if (keyIt->second.Unsorted().Add(id, this->opts_.IsPK() ? IdSet::Ordered : IdSet::Auto, this->sortedIdxCount_)) { this->isBuilt_ = false; - if (this->cache_) this->cache_.reset(); + this->cache_.reset(); clearCache = true; } this->tracker_.markUpdated(this->idx_map, keyIt); @@ -56,10 +56,6 @@ SelectKeyResults IndexOrdered::SelectKey(const VariantArray &keys, CondType c return IndexUnordered::SelectKey(keys, condition, sortId, opts, ctx, rdxCtx); } - if (keys.size() < 1) { - throw Error(errParams, "For condition required at least 1 argument, but provided 0"); - } - SelectKeyResult res; auto startIt = this->idx_map.begin(); auto endIt = this->idx_map.end(); @@ -81,7 +77,6 @@ SelectKeyResults IndexOrdered::SelectKey(const VariantArray &keys, CondType c if (startIt == this->idx_map.end()) startIt = this->idx_map.upper_bound(static_cast(key1)); break; case CondRange: { - if (keys.size() != 2) throw Error(errParams, "For ranged query reuqired 2 arguments, but provided %d", keys.size()); const auto &key2 = keys[1]; startIt = this->idx_map.find(static_cast(key1)); @@ -207,18 +202,20 @@ IndexIterator::Ptr IndexOrdered::CreateIterator() const { } template -static std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +static std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { switch (idef.Type()) { case IndexIntBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexInt64BTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexStrBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexDoubleBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexCompositeBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), cacheCfg); case IndexStrHash: case IndexIntHash: case IndexInt64Hash: @@ -242,9 +239,11 @@ static std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType } // NOLINTBEGIN(*cplusplus.NewDeleteLeaks) -std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { - return (idef.opts_.IsPK() || idef.opts_.IsDense()) ? IndexOrdered_New(idef, std::move(payloadType), fields) - : IndexOrdered_New(idef, std::move(payloadType), fields); +std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { + return (idef.opts_.IsPK() || idef.opts_.IsDense()) + ? IndexOrdered_New(idef, std::move(payloadType), std::move(fields), cacheCfg) + : IndexOrdered_New(idef, std::move(payloadType), std::move(fields), cacheCfg); } // NOLINTEND(*cplusplus.NewDeleteLeaks) diff --git a/cpp_src/core/index/indexordered.h b/cpp_src/core/index/indexordered.h index e8dc6d229..a12928024 100644 --- a/cpp_src/core/index/indexordered.h +++ b/cpp_src/core/index/indexordered.h @@ -10,18 +10,19 @@ class IndexOrdered : public IndexUnordered { using ref_type = typename IndexUnordered::ref_type; using key_type = typename IndexUnordered::key_type; - IndexOrdered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : IndexUnordered(idef, std::move(payloadType), fields) {} + IndexOrdered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, const NamespaceCacheConfigData &cacheCfg) + : IndexUnordered(idef, std::move(payloadType), std::move(fields), cacheCfg) {} SelectKeyResults SelectKey(const VariantArray &keys, CondType condition, SortType stype, Index::SelectOpts opts, const BaseFunctionCtx::Ptr &ctx, const RdxContext &) override; Variant Upsert(const Variant &key, IdType id, bool &clearCache) override; void MakeSortOrders(UpdateSortedContext &ctx) override; IndexIterator::Ptr CreateIterator() const override; - std::unique_ptr Clone() const override { return std::unique_ptr{new IndexOrdered(*this)}; } + std::unique_ptr Clone() const override { return std::make_unique>(*this); } bool IsOrdered() const noexcept override { return true; } }; -std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); +std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg); } // namespace reindexer diff --git a/cpp_src/core/index/indexstore.cc b/cpp_src/core/index/indexstore.cc index 27fe6239f..df95f979c 100644 --- a/cpp_src/core/index/indexstore.cc +++ b/cpp_src/core/index/indexstore.cc @@ -7,8 +7,8 @@ namespace reindexer { template <> -IndexStore::IndexStore(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Index(idef, std::move(payloadType), fields) { +IndexStore::IndexStore(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Index(idef, std::move(payloadType), std::move(fields)) { keyType_ = selectKeyType_ = KeyValueType::Double{}; opts_.Array(true); } @@ -68,8 +68,8 @@ Variant IndexStore::Upsert(const Variant &key, IdType /*id*/, bool template Variant IndexStore::Upsert(const Variant &key, IdType id, bool & /*clearCache*/) { - if (!opts_.IsArray() && !opts_.IsDense() && !opts_.IsSparse() && key.Type().Is()) { - idx_data.resize(std::max(id + 1, int(idx_data.size()))); + if (!opts_.IsArray() && !opts_.IsDense() && !opts_.IsSparse() && !key.Type().Is()) { + idx_data.resize(std::max(id + 1, IdType(idx_data.size()))); idx_data[id] = static_cast(key); } return Variant(key); @@ -106,43 +106,8 @@ SelectKeyResults IndexStore::SelectKey(const VariantArray &keys, CondType con if (condition == CondAny && !this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct) throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes"); - // TODO: it may be necessary to remove or change this switch after QueryEntry refactoring - switch (condition) { - case CondAny: - if (!this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct) { - throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes"); - } - break; - case CondEmpty: - if (!this->opts_.IsArray() && !this->opts_.IsSparse()) { - throw Error(errParams, "The 'is NULL' condition is suported only by 'sparse' or 'array' indexes"); - } - break; - case CondAllSet: - case CondSet: - case CondEq: - break; - case CondRange: - case CondDWithin: - if (keys.size() != 2) { - throw Error(errParams, "For condition %s required exactly 2 arguments, but provided %d", CondTypeToStr(condition), - keys.size()); - } - break; - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - if (keys.size() != 1) { - throw Error(errParams, "For condition %s required exactly 1 argument, but provided %d", CondTypeToStr(condition), - keys.size()); - } - break; - } - - res.comparators_.push_back(Comparator(condition, KeyType(), keys, opts_.IsArray(), sopts.distinct, payloadType_, fields_, - idx_data.size() ? idx_data.data() : nullptr, opts_.collateOpts_)); + res.comparators_.emplace_back(condition, KeyType(), keys, opts_.IsArray(), bool(sopts.distinct), payloadType_, Fields(), + idx_data.size() ? idx_data.data() : nullptr, opts_.collateOpts_); return SelectKeyResults(std::move(res)); } @@ -183,20 +148,20 @@ void IndexStore::AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue &q) (void)q; } -std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { switch (idef.Type()) { case IndexBool: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields)); case IndexIntStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields)); case IndexInt64Store: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields)); case IndexDoubleStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields)); case IndexStrStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields)); case IndexUuidStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields)); case IndexStrHash: case IndexStrBTree: case IndexIntBTree: diff --git a/cpp_src/core/index/indexstore.h b/cpp_src/core/index/indexstore.h index 832c11a4f..6c3a1b4f8 100644 --- a/cpp_src/core/index/indexstore.h +++ b/cpp_src/core/index/indexstore.h @@ -8,7 +8,8 @@ namespace reindexer { template class IndexStore : public Index { public: - IndexStore(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) : Index(idef, std::move(payloadType), fields) { + IndexStore(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Index(idef, std::move(payloadType), std::move(fields)) { static T a; keyType_ = selectKeyType_ = Variant(a).Type(); } @@ -21,13 +22,14 @@ class IndexStore : public Index { const BaseFunctionCtx::Ptr &ctx, const RdxContext &) override; void Commit() override; void UpdateSortedIds(const UpdateSortedContext & /*ctx*/) override {} - std::unique_ptr Clone() const override { return std::unique_ptr{new IndexStore(*this)}; } + std::unique_ptr Clone() const override { return std::make_unique>(*this); } IndexMemStat GetMemStat(const RdxContext &) override; bool HoldsStrings() const noexcept override { return std::is_same_v || std::is_same_v; } void Dump(std::ostream &os, std::string_view step = " ", std::string_view offset = "") const override { dump(os, step, offset); } virtual void AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue &) override; virtual bool IsDestroyPartSupported() const noexcept override { return true; } virtual bool IsUuid() const noexcept override final { return std::is_same_v; } + virtual void ReconfigureCache(const NamespaceCacheConfigData &) override {} template struct HasAddTask : std::false_type {}; @@ -46,8 +48,8 @@ class IndexStore : public Index { }; template <> -IndexStore::IndexStore(const IndexDef &, PayloadType, const FieldsSet &); +IndexStore::IndexStore(const IndexDef &, PayloadType &&, FieldsSet &&); -std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields_); +std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&); } // namespace reindexer diff --git a/cpp_src/core/index/indextext/fastindextext.cc b/cpp_src/core/index/indextext/fastindextext.cc index 36db2dc49..12cbdcae9 100644 --- a/cpp_src/core/index/indextext/fastindextext.cc +++ b/cpp_src/core/index/indextext/fastindextext.cc @@ -125,7 +125,7 @@ IndexMemStat FastIndexText::GetMemStat(const RdxContext &ctx) { contexted_shared_lock lck(this->mtx_, &ctx); ret.fulltextSize = this->holder_->GetMemStat(); - if (this->cache_ft_) ret.idsetCache = this->cache_ft_->GetMemStat(); + ret.idsetCache = this->cache_ft_ ? this->cache_ft_->GetMemStat() : LRUCacheMemStat(); return ret; } @@ -135,7 +135,7 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery &&dsl, bool inTr fctx->GetData()->extraWordSymbols_ = this->getConfig()->extraWordSymbols; fctx->GetData()->isWordPositions_ = true; - auto mergeData = this->holder_->Select(std::move(dsl), this->fields_.size(), fctx->NeedArea(), getConfig()->maxAreasInDoc, + auto mergeData = this->holder_->Select(std::move(dsl), this->Fields().size(), fctx->NeedArea(), getConfig()->maxAreasInDoc, inTransaction, std::move(statuses.statuses), useExternSt, rdxCtx); // convert vids(uniq documents id) to ids (real ids) IdSet::Ptr mergedIds = make_intrusive>(); @@ -232,7 +232,7 @@ void FastIndexText::commitFulltextImpl() { } auto tm1 = high_resolution_clock::now(); - this->holder_->Process(this->fields_.size(), !this->opts_.IsDense()); + this->holder_->Process(this->Fields().size(), !this->opts_.IsDense()); if (this->holder_->NeedClear(this->tracker_.isCompleteUpdated())) { this->tracker_.clear(); } @@ -373,12 +373,15 @@ reindexer::FtPreselectT FastIndexText::FtPreselect(const RdxContext &rdxCtx) std::vector(holder_->rowId2Vdoc_.size(), false), &holder_->rowId2Vdoc_}; } -std::unique_ptr FastIndexText_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr FastIndexText_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { switch (idef.Type()) { case IndexFastFT: - return std::unique_ptr{new FastIndexText>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexCompositeFastFT: - return std::unique_ptr{new FastIndexText>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexStrHash: case IndexStrBTree: case IndexIntBTree: diff --git a/cpp_src/core/index/indextext/fastindextext.h b/cpp_src/core/index/indextext/fastindextext.h index 5a6ebb50d..1a2bb913a 100644 --- a/cpp_src/core/index/indextext/fastindextext.h +++ b/cpp_src/core/index/indextext/fastindextext.h @@ -2,8 +2,6 @@ #include "core/ft/config/ftfastconfig.h" #include "core/ft/ft_fast/dataholder.h" -#include "core/ft/ft_fast/dataprocessor.h" -#include "core/ft/typos.h" #include "indextext.h" namespace reindexer { @@ -22,10 +20,11 @@ class FastIndexText : public IndexText { this->CommitFulltext(); } - FastIndexText(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) : Base(idef, std::move(payloadType), fields) { + FastIndexText(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, const NamespaceCacheConfigData& cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields), cacheCfg) { initConfig(); } - std::unique_ptr Clone() const override { return std::unique_ptr{new FastIndexText(*this)}; } + std::unique_ptr Clone() const override { return std::make_unique>(*this); } IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, const RdxContext&) override final; IndexMemStat GetMemStat(const RdxContext&) override final; @@ -40,7 +39,7 @@ class FastIndexText : public IndexText { reindexer::FtPreselectT FtPreselect(const RdxContext& rdxCtx) override final; bool EnablePreselectBeforeFt() const override final { return getConfig()->enablePreselectBeforeFt; } -protected: +private: void commitFulltextImpl() override final; FtFastConfig* getConfig() const noexcept { return dynamic_cast(this->cfg_.get()); } void initConfig(const FtFastConfig* = nullptr); @@ -53,6 +52,7 @@ class FastIndexText : public IndexText { std::unique_ptr holder_; }; -std::unique_ptr FastIndexText_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); +std::unique_ptr FastIndexText_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, + const NamespaceCacheConfigData& cacheCfg); } // namespace reindexer diff --git a/cpp_src/core/index/indextext/fieldsgetter.h b/cpp_src/core/index/indextext/fieldsgetter.h index 791932b27..3af05f79f 100644 --- a/cpp_src/core/index/indextext/fieldsgetter.h +++ b/cpp_src/core/index/indextext/fieldsgetter.h @@ -1,6 +1,5 @@ #pragma once #include "core/ft/usingcontainer.h" -#include "core/index/payload_map.h" #include "core/payload/fieldsset.h" #include "vendor/utf8cpp/utf8.h" diff --git a/cpp_src/core/index/indextext/fuzzyindextext.cc b/cpp_src/core/index/indextext/fuzzyindextext.cc index 339ac1252..1e2344cf0 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.cc +++ b/cpp_src/core/index/indextext/fuzzyindextext.cc @@ -56,7 +56,7 @@ void FuzzyIndexText::commitFulltextImpl() { } template -void FuzzyIndexText::CreateConfig(const FtFuzzyConfig* cfg) { +void FuzzyIndexText::createConfig(const FtFuzzyConfig* cfg) { if (cfg) { this->cfg_.reset(new FtFuzzyConfig(*cfg)); return; @@ -65,13 +65,15 @@ void FuzzyIndexText::CreateConfig(const FtFuzzyConfig* cfg) { this->cfg_->parse(this->opts_.config, this->ftFields_); } -std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) { +std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, + const NamespaceCacheConfigData& cacheCfg) { switch (idef.Type()) { case IndexFuzzyFT: - return std::unique_ptr{new FuzzyIndexText>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexCompositeFuzzyFT: - return std::unique_ptr{ - new FuzzyIndexText>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), + std::move(fields), cacheCfg); case IndexStrHash: case IndexStrBTree: case IndexIntBTree: diff --git a/cpp_src/core/index/indextext/fuzzyindextext.h b/cpp_src/core/index/indextext/fuzzyindextext.h index d1a94780a..0c5d90846 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.h +++ b/cpp_src/core/index/indextext/fuzzyindextext.h @@ -1,9 +1,6 @@ #pragma once -#include "core/ft/config/ftfastconfig.h" #include "core/ft/ft_fuzzy/searchengine.h" -#include "core/ft/ftsetcashe.h" -#include "core/ft/idrelset.h" #include "indextext.h" namespace reindexer { @@ -13,10 +10,11 @@ class FuzzyIndexText : public IndexText { using Base = IndexText; public: - FuzzyIndexText(const FuzzyIndexText& other) : Base(other) { CreateConfig(other.getConfig()); } + FuzzyIndexText(const FuzzyIndexText& other) : Base(other) { createConfig(other.getConfig()); } - FuzzyIndexText(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) : Base(idef, std::move(payloadType), fields) { - CreateConfig(); + FuzzyIndexText(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, const NamespaceCacheConfigData& cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields), cacheCfg) { + createConfig(); } SelectKeyResults SelectKey(const VariantArray& /*keys*/, CondType, Index::SelectOpts, const BaseFunctionCtx::Ptr&, FtPreselectT&&, @@ -24,7 +22,7 @@ class FuzzyIndexText : public IndexText { assertrx(0); abort(); } - std::unique_ptr Clone() const override final { return std::unique_ptr{new FuzzyIndexText(*this)}; } + std::unique_ptr Clone() const override final { return std::make_unique>(*this); } IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, const RdxContext&) override final; Variant Upsert(const Variant& key, IdType id, bool& clearCache) override final { @@ -43,12 +41,13 @@ class FuzzyIndexText : public IndexText { protected: void commitFulltextImpl() override final; FtFuzzyConfig* getConfig() const noexcept { return dynamic_cast(this->cfg_.get()); } - void CreateConfig(const FtFuzzyConfig* cfg = nullptr); + void createConfig(const FtFuzzyConfig* cfg = nullptr); search_engine::SearchEngine engine_; std::vector vdocs_; }; -std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); +std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, + const NamespaceCacheConfigData& cacheCfg); } // namespace reindexer diff --git a/cpp_src/core/index/indextext/indextext.cc b/cpp_src/core/index/indextext/indextext.cc index 4a8a646db..4ef8d42ee 100644 --- a/cpp_src/core/index/indextext/indextext.cc +++ b/cpp_src/core/index/indextext/indextext.cc @@ -9,7 +9,11 @@ namespace reindexer { template -IndexText::IndexText(const IndexText &other) : IndexUnordered(other), cache_ft_(std::make_shared()) { +IndexText::IndexText(const IndexText &other) + : IndexUnordered(other), + cache_ft_(std::make_unique(other.cacheMaxSize_, other.hitsToCache_)), + cacheMaxSize_(other.cacheMaxSize_), + hitsToCache_(other.hitsToCache_) { initSearchers(); } // Generic implemetation for string index @@ -19,16 +23,17 @@ void IndexText::initSearchers() { size_t jsonPathIdx = 0; if (this->payloadType_) { - for (unsigned i = 0; i < this->fields_.size(); i++) { - auto fieldIdx = this->fields_[i]; + const auto &fields = this->Fields(); + for (unsigned i = 0, s = fields.size(); i < s; i++) { + auto fieldIdx = fields[i]; if (fieldIdx == IndexValueType::SetByJsonPath) { - assertrx(jsonPathIdx < this->fields_.getJsonPathsLength()); - ftFields_.emplace(this->fields_.getJsonPath(jsonPathIdx++), i); + assertrx(jsonPathIdx < fields.getJsonPathsLength()); + ftFields_.emplace(fields.getJsonPath(jsonPathIdx++), i); } else { ftFields_.emplace(this->payloadType_->Field(fieldIdx).Name(), i); } } - if rx_unlikely (ftFields_.size() != this->fields_.size()) { + if rx_unlikely (ftFields_.size() != fields.size()) { throw Error(errParams, "Composite fulltext index '%s' contains duplicated fields", this->name_); } if rx_unlikely (ftFields_.size() > kMaxFtCompositeFields) { @@ -55,6 +60,18 @@ void IndexText::SetOpts(const IndexOpts &opts) { } } +template +void IndexText::ReconfigureCache(const NamespaceCacheConfigData &cacheCfg) { + if (cacheMaxSize_ != cacheCfg.ftIdxCacheSize || hitsToCache_ != cacheCfg.ftIdxHitsToCache) { + cacheMaxSize_ = cacheCfg.ftIdxCacheSize; + hitsToCache_ = cacheCfg.ftIdxHitsToCache; + if (cache_ft_) { + cache_ft_ = std::make_unique(cacheMaxSize_, hitsToCache_); + } + } + Base::ReconfigureCache(cacheCfg); +} + template FtCtx::Ptr IndexText::prepareFtCtx(const BaseFunctionCtx::Ptr &ctx) { FtCtx::Ptr ftctx = reindexer::reinterpret_pointer_cast(ctx); @@ -178,7 +195,7 @@ SelectKeyResults IndexText::SelectKey(const VariantArray &keys, CondType cond template FieldsGetter IndexText::Getter() { - return FieldsGetter(this->fields_, this->payloadType_, this->KeyType()); + return FieldsGetter(this->Fields(), this->payloadType_, this->KeyType()); } template class IndexText>; diff --git a/cpp_src/core/index/indextext/indextext.h b/cpp_src/core/index/indextext/indextext.h index d09c61a26..b1458227d 100644 --- a/cpp_src/core/index/indextext/indextext.h +++ b/cpp_src/core/index/indextext/indextext.h @@ -8,7 +8,6 @@ #include "core/ft/ftsetcashe.h" #include "core/index/indexunordered.h" #include "core/selectfunc/ctx/ftctx.h" -#include "estl/fast_hash_map.h" #include "estl/shared_mutex.h" #include "fieldsgetter.h" @@ -20,8 +19,11 @@ class IndexText : public IndexUnordered { public: IndexText(const IndexText& other); - IndexText(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) - : IndexUnordered(idef, std::move(payloadType), fields), cache_ft_(std::make_shared()) { + IndexText(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, const NamespaceCacheConfigData& cacheCfg) + : IndexUnordered(idef, std::move(payloadType), std::move(fields), cacheCfg), + cache_ft_(std::make_unique(cacheCfg.ftIdxCacheSize, cacheCfg.ftIdxHitsToCache)), + cacheMaxSize_(cacheCfg.ftIdxCacheSize), + hitsToCache_(cacheCfg.ftIdxHitsToCache) { this->selectKeyType_ = KeyValueType::String{}; initSearchers(); } @@ -39,7 +41,7 @@ class IndexText : public IndexUnordered { // Rebuild will be done on first select } void CommitFulltext() override final { - cache_ft_ = std::make_shared(); + cache_ft_ = std::make_unique(cacheMaxSize_, hitsToCache_); commitFulltextImpl(); this->isBuilt_ = true; } @@ -51,7 +53,8 @@ class IndexText : public IndexUnordered { } void ClearCache(const std::bitset& s) override { Base::ClearCache(s); } void MarkBuilt() noexcept override { assertrx(0); } - bool IsFulltext() const noexcept override { return true; } + bool IsFulltext() const noexcept override final { return true; } + void ReconfigureCache(const NamespaceCacheConfigData& cacheCfg) override final; protected: using Mutex = MarkedMutex; @@ -66,7 +69,9 @@ class IndexText : public IndexUnordered { void initSearchers(); FieldsGetter Getter(); - std::shared_ptr cache_ft_; + std::unique_ptr cache_ft_; + size_t cacheMaxSize_; + uint32_t hitsToCache_; RHashMap ftFields_; std::unique_ptr cfg_; diff --git a/cpp_src/core/index/indexunordered.cc b/cpp_src/core/index/indexunordered.cc index b311fe25f..06af94780 100644 --- a/cpp_src/core/index/indexunordered.cc +++ b/cpp_src/core/index/indexunordered.cc @@ -3,7 +3,6 @@ #include "core/index/payload_map.h" #include "core/index/string_map.h" #include "core/indexdef.h" -#include "core/keyvalue/uuid.h" #include "core/rdxcontext.h" #include "rtree/greenesplitter.h" #include "rtree/linearsplitter.h" @@ -18,55 +17,95 @@ namespace reindexer { constexpr int kMaxIdsForDistinct = 500; template -IndexUnordered::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map() { +IndexUnordered::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) { static_assert(!(is_str_map_v || is_payload_map_v)); } template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(idef.opts_.collateOpts_), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(idef.opts_.collateOpts_), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(idef.opts_.collateOpts_), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(idef.opts_.collateOpts_), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(idef.opts_.collateOpts_), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, + FieldsSet &&fields, const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, + FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) + : Base(idef, std::move(payloadType), std::move(fields)), + idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}), + cacheMaxSize_(cacheCfg.idxIdsetCacheSize), + hitsToCache_(cacheCfg.idxIdsetHitsToCache) {} template bool IndexUnordered::HoldsStrings() const noexcept { @@ -79,7 +118,13 @@ bool IndexUnordered::HoldsStrings() const noexcept { template IndexUnordered::IndexUnordered(const IndexUnordered &other) - : Base(other), idx_map(other.idx_map), cache_(nullptr), empty_ids_(other.empty_ids_), tracker_(other.tracker_) {} + : Base(other), + idx_map(other.idx_map), + cache_(nullptr), + cacheMaxSize_(other.cacheMaxSize_), + hitsToCache_(other.hitsToCache_), + empty_ids_(other.empty_ids_), + tracker_(other.tracker_) {} template size_t heap_size(const key_type & /*kt*/) { @@ -130,7 +175,7 @@ Variant IndexUnordered::Upsert(const Variant &key, IdType id, bool &clearCach // reset cache if (key.Type().Is()) { // TODO maybe error or default value if the index is not sparse if (this->empty_ids_.Unsorted().Add(id, IdSet::Auto, this->sortedIdxCount_)) { - if (cache_) cache_.reset(); + cache_.reset(); clearCache = true; this->isBuilt_ = false; } @@ -146,7 +191,7 @@ Variant IndexUnordered::Upsert(const Variant &key, IdType id, bool &clearCach } if (keyIt->second.Unsorted().Add(id, this->opts_.IsPK() ? IdSet::Ordered : IdSet::Auto, this->sortedIdxCount_)) { - if (cache_) cache_.reset(); + cache_.reset(); clearCache = true; this->isBuilt_ = false; } @@ -168,7 +213,7 @@ void IndexUnordered::Delete(const Variant &key, IdType id, StringsHolder &str delcnt = this->empty_ids_.Unsorted().Erase(id); assertrx(delcnt); this->isBuilt_ = false; - if (cache_) cache_.reset(); + cache_.reset(); clearCache = true; return; } @@ -180,12 +225,12 @@ void IndexUnordered::Delete(const Variant &key, IdType id, StringsHolder &str delcnt = keyIt->second.Unsorted().Erase(id); (void)delcnt; this->isBuilt_ = false; - if (cache_) cache_.reset(); + cache_.reset(); clearCache = true; // TODO: we have to implement removal of composite indexes (doesn't work right now) assertf(this->opts_.IsArray() || this->Opts().IsSparse() || delcnt, "Delete unexists id from index '%s' id=%d,key=%s (%s)", this->name_, - id, key.As(this->payloadType_, this->fields_), - Variant(keyIt->first).As(this->payloadType_, this->fields_)); + id, key.As(this->payloadType_, this->Fields()), + Variant(keyIt->first).As(this->payloadType_, this->Fields())); if (keyIt->second.Unsorted().IsEmpty()) { this->tracker_.markDeleted(keyIt); @@ -345,7 +390,7 @@ template void IndexUnordered::Commit() { this->empty_ids_.Unsorted().Commit(); - if (!cache_) cache_.reset(new IdSetCache()); + if (!cache_) cache_.reset(new IdSetCache(cacheMaxSize_, hitsToCache_)); if (!tracker_.isUpdated()) return; @@ -433,18 +478,33 @@ void IndexUnordered::AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue (void)q; } +template +void IndexUnordered::ReconfigureCache(const NamespaceCacheConfigData &cacheCfg) { + if (cacheMaxSize_ != cacheCfg.idxIdsetCacheSize || hitsToCache_ != cacheCfg.idxIdsetHitsToCache) { + cacheMaxSize_ = cacheCfg.idxIdsetCacheSize; + hitsToCache_ = cacheCfg.idxIdsetHitsToCache; + if (cache_) { + cache_.reset(new IdSetCache(cacheMaxSize_, hitsToCache_)); + } + } +} + template -static std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +static std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { switch (idef.Type()) { case IndexIntHash: - return std::unique_ptr{new IndexUnordered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexInt64Hash: - return std::unique_ptr{ - new IndexUnordered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), + std::move(fields), cacheCfg); case IndexStrHash: - return std::unique_ptr{new IndexUnordered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexCompositeHash: - return std::unique_ptr{new IndexUnordered>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexStrBTree: case IndexIntBTree: case IndexInt64BTree: @@ -469,9 +529,11 @@ static std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadTy } // NOLINTBEGIN(*cplusplus.NewDeleteLeaks) -std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { - return (idef.opts_.IsPK() || idef.opts_.IsDense()) ? IndexUnordered_New(idef, std::move(payloadType), fields) - : IndexUnordered_New(idef, std::move(payloadType), fields); +std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { + return (idef.opts_.IsPK() || idef.opts_.IsDense()) + ? IndexUnordered_New(idef, std::move(payloadType), std::move(fields), cacheCfg) + : IndexUnordered_New(idef, std::move(payloadType), std::move(fields), cacheCfg); } // NOLINTEND(*cplusplus.NewDeleteLeaks) diff --git a/cpp_src/core/index/indexunordered.h b/cpp_src/core/index/indexunordered.h index 05cc89d95..11b8fbe74 100644 --- a/cpp_src/core/index/indexunordered.h +++ b/cpp_src/core/index/indexunordered.h @@ -26,7 +26,7 @@ class IndexUnordered : public IndexStore> { typename T::key_type>::type>::type>::type; using key_type = StoreIndexKeyType; - IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); + IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, const NamespaceCacheConfigData &cacheCfg); IndexUnordered(const IndexUnordered &other); Variant Upsert(const Variant &key, IdType id, bool &chearCache) override; @@ -35,7 +35,7 @@ class IndexUnordered : public IndexStore> { const BaseFunctionCtx::Ptr &ctx, const RdxContext &) override; void Commit() override; void UpdateSortedIds(const UpdateSortedContext &) override; - std::unique_ptr Clone() const override { return std::unique_ptr{new IndexUnordered(*this)}; } + std::unique_ptr Clone() const override { return std::make_unique>(*this); } IndexMemStat GetMemStat(const RdxContext &) override; size_t Size() const noexcept override final { return idx_map.size(); } void SetSortedIdxCount(int sortedIdxCount) override; @@ -49,6 +49,7 @@ class IndexUnordered : public IndexStore> { void AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue &) override; bool IsDestroyPartSupported() const noexcept override { return true; } + void ReconfigureCache(const NamespaceCacheConfigData &cacheCfg) override; protected: bool tryIdsetCache(const VariantArray &keys, CondType condition, SortType sortId, @@ -60,6 +61,8 @@ class IndexUnordered : public IndexStore> { T idx_map; // Merged idsets cache atomic_unique_ptr cache_; + size_t cacheMaxSize_; + uint32_t hitsToCache_; // Empty ids Index::KeyEntry empty_ids_; // Tracker of updates @@ -70,8 +73,9 @@ class IndexUnordered : public IndexStore> { void dump(S &os, std::string_view step, std::string_view offset) const; }; -constexpr inline unsigned maxSelectivityPercentForIdset() noexcept { return 30u; } +constexpr unsigned maxSelectivityPercentForIdset() noexcept { return 30u; } -std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); +std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg); } // namespace reindexer diff --git a/cpp_src/core/index/payload_map.h b/cpp_src/core/index/payload_map.h index 91132989f..357b671a3 100644 --- a/cpp_src/core/index/payload_map.h +++ b/cpp_src/core/index/payload_map.h @@ -12,7 +12,7 @@ namespace reindexer { class PayloadValueWithHash : public PayloadValue { public: PayloadValueWithHash() noexcept : PayloadValue() {} - PayloadValueWithHash(PayloadValue pv, const PayloadType &pt, const FieldsSet &fields) + PayloadValueWithHash(PayloadValue &&pv, const PayloadType &pt, const FieldsSet &fields) : PayloadValue(std::move(pv)), hash_(ConstPayload(pt, *static_cast(this)).GetHash(fields)) {} PayloadValueWithHash(const PayloadValueWithHash &o) noexcept : PayloadValue(o), hash_(o.hash_) {} PayloadValueWithHash(PayloadValueWithHash &&o) noexcept : PayloadValue(std::move(o)), hash_(o.hash_) {} @@ -29,7 +29,8 @@ class PayloadValueWithHash : public PayloadValue { struct equal_composite { using is_transparent = void; - equal_composite(PayloadType type, const FieldsSet &fields) : type_(std::move(type)), fields_(fields) {} + template + equal_composite(PT &&type, FS &&fields) : type_(std::forward(type)), fields_(std::forward(fields)) {} bool operator()(const PayloadValue &lhs, const PayloadValue &rhs) const { assertrx(type_); return ConstPayload(type_, lhs).IsEQ(rhs, fields_); @@ -50,7 +51,8 @@ struct equal_composite { FieldsSet fields_; }; struct hash_composite { - hash_composite(PayloadType type, const FieldsSet &fields) : type_(std::move(type)), fields_(fields) {} + template + hash_composite(PT &&type, FS &&fields) : type_(std::forward(type)), fields_(std::forward(fields)) {} size_t operator()(const PayloadValueWithHash &s) const { return s.GetHash(); } size_t operator()(const PayloadValue &s) const { assertrx(type_); @@ -61,7 +63,7 @@ struct hash_composite { }; struct less_composite { - less_composite(PayloadType type, const FieldsSet &fields) : type_(std::move(type)), fields_(fields) {} + less_composite(PayloadType &&type, FieldsSet &&fields) : type_(std::move(type)), fields_(std::move(fields)) {} bool operator()(const PayloadValue &lhs, const PayloadValue &rhs) const { assertrx(type_); assertrx(!lhs.IsFree()); @@ -78,7 +80,7 @@ class payload_str_fields_helper; template <> class payload_str_fields_helper { protected: - payload_str_fields_helper(PayloadType payloadType, const FieldsSet &fields) : payload_type_(std::move(payloadType)) { + payload_str_fields_helper(PayloadType &&payloadType, const FieldsSet &fields) : payload_type_(std::move(payloadType)) { if (fields.getTagsPathsLength() || fields.getJsonPathsLength()) { str_fields_.push_back(0); } @@ -153,13 +155,13 @@ class unordered_payload_map static_assert(std::is_nothrow_move_constructible>::value, "Nothrow movebale key and value required"); - unordered_payload_map(size_t size, PayloadType pt, const FieldsSet &f) - : base_hash_map(size, hash_composite(pt, f), equal_composite(pt, f)), - payload_str_fields_helper(pt, f), + unordered_payload_map(size_t size, PayloadType &&pt, FieldsSet &&f) + : base_hash_map(size, hash_composite(PayloadType{pt}, FieldsSet{f}), equal_composite(PayloadType{pt}, FieldsSet{f})), + payload_str_fields_helper(PayloadType{pt}, f), payloadType_(std::move(pt)), - fields_(f) {} + fields_(std::move(f)) {} - unordered_payload_map(PayloadType pt, const FieldsSet &f) : unordered_payload_map(1000, std::move(pt), f) {} + unordered_payload_map(PayloadType &&pt, FieldsSet &&f) : unordered_payload_map(1000, std::move(pt), std::move(f)) {} unordered_payload_map(const unordered_payload_map &other) : base_hash_map(other), payload_str_fields_helper(other), payloadType_(other.payloadType_), fields_(other.fields_) { @@ -185,7 +187,7 @@ class unordered_payload_map } template std::pair emplace(const PayloadValue &pl, V &&v) { - PayloadValueWithHash key(pl, payloadType_, fields_); + PayloadValueWithHash key(PayloadValue{pl}, payloadType_, fields_); auto res = base_hash_map::emplace(std::move(key), std::forward(v)); if (res.second) this->add_ref(res.first->first); return res; @@ -207,7 +209,7 @@ class unordered_payload_map } T1 &operator[](const PayloadValue &k) { - PayloadValueWithHash key(k, payloadType_, fields_); + PayloadValueWithHash key(PayloadValue{k}, payloadType_, fields_); return base_hash_map::operator[](std::move(key)); } T1 &operator[](PayloadValue &&k) { @@ -247,7 +249,8 @@ class payload_map : private btree::btree_map, using payload_str_fields_helper::have_str_fields; payload_map(PayloadType payloadType, const FieldsSet &fields) - : base_tree_map(less_composite(payloadType, fields)), payload_str_fields_helper(std::move(payloadType), fields) {} + : base_tree_map(less_composite(PayloadType{payloadType}, FieldsSet{fields})), + payload_str_fields_helper(std::move(payloadType), fields) {} payload_map(const payload_map &other) : base_tree_map(other), payload_str_fields_helper(other) { for (auto &item : *this) this->add_ref(const_cast(item.first)); } diff --git a/cpp_src/core/index/rtree/indexrtree.cc b/cpp_src/core/index/rtree/indexrtree.cc index 555db9012..c285a768d 100644 --- a/cpp_src/core/index/rtree/indexrtree.cc +++ b/cpp_src/core/index/rtree/indexrtree.cc @@ -78,7 +78,7 @@ void IndexRTree::Upsert(VariantArra if (keyIt->second.Unsorted().Add(id, this->opts_.IsPK() ? IdSet::Ordered : IdSet::Auto, this->sortedIdxCount_)) { this->isBuilt_ = false; // reset cache - if (this->cache_) this->cache_.reset(); + this->cache_.reset(); clearCache = true; } this->tracker_.markUpdated(this->idx_map, keyIt); @@ -99,7 +99,7 @@ void IndexRTree::Delete(const Varia const Point point = static_cast(keys); typename Map::iterator keyIt = this->idx_map.find(point); if (keyIt == this->idx_map.end()) return; - if (this->cache_) this->cache_.reset(); + this->cache_.reset(); clearCache = true; this->isBuilt_ = false; @@ -108,8 +108,8 @@ void IndexRTree::Delete(const Varia (void)delcnt; // TODO: we have to implement removal of composite indexes (doesn't work right now) assertf(this->Opts().IsSparse() || delcnt, "Delete unexists id from index '%s' id=%d,key=%s (%s)", this->name_, id, - Variant(keys).template As(this->payloadType_, this->fields_), - Variant(keyIt->first).As(this->payloadType_, this->fields_)); + Variant(keys).template As(this->payloadType_, this->Fields()), + Variant(keyIt->first).As(this->payloadType_, this->Fields())); if (keyIt->second.Unsorted().IsEmpty()) { this->tracker_.markDeleted(keyIt); @@ -120,41 +120,44 @@ void IndexRTree::Delete(const Varia } } -std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { switch (idef.opts_.RTreeType()) { case IndexOpts::Linear: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { - return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), + std::move(fields), cacheCfg); } else { - return std::unique_ptr{new IndexRTree(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields), + cacheCfg); } case IndexOpts::Quadratic: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { - return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; - } else { - return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), + std::move(fields), cacheCfg); } + return std::make_unique>(idef, std::move(payloadType), std::move(fields), + cacheCfg); case IndexOpts::Greene: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { - return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), + std::move(fields), cacheCfg); } else { - return std::unique_ptr{new IndexRTree(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields), + cacheCfg); } case IndexOpts::RStar: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { - return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), + std::move(fields), cacheCfg); } else { - return std::unique_ptr{new IndexRTree(idef, std::move(payloadType), fields)}; + return std::make_unique>(idef, std::move(payloadType), std::move(fields), + cacheCfg); } - default: - assertrx(0); - abort(); } + + assertrx(0); + std::abort(); } template class IndexRTree; diff --git a/cpp_src/core/index/rtree/indexrtree.h b/cpp_src/core/index/rtree/indexrtree.h index effdbd04d..9d4b23986 100644 --- a/cpp_src/core/index/rtree/indexrtree.h +++ b/cpp_src/core/index/rtree/indexrtree.h @@ -11,8 +11,8 @@ class IndexRTree : public IndexUnordered; public: - IndexRTree(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : IndexUnordered{idef, std::move(payloadType), fields} {} + IndexRTree(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, const NamespaceCacheConfigData &cacheCfg) + : IndexUnordered{idef, std::move(payloadType), std::move(fields), cacheCfg} {} SelectKeyResults SelectKey(const VariantArray &keys, CondType, SortType, Index::SelectOpts, const BaseFunctionCtx::Ptr &, const RdxContext &) override; @@ -21,9 +21,10 @@ class IndexRTree : public IndexUnordered::Delete; void Delete(const VariantArray &keys, IdType id, StringsHolder &, bool &clearCache) override; - std::unique_ptr Clone() const override { return std::unique_ptr{new IndexRTree(*this)}; } + std::unique_ptr Clone() const override { return std::make_unique(*this); } }; -std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); +std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg); } // namespace reindexer diff --git a/cpp_src/core/index/string_map.h b/cpp_src/core/index/string_map.h index 98114e386..ae60f9221 100644 --- a/cpp_src/core/index/string_map.h +++ b/cpp_src/core/index/string_map.h @@ -2,15 +2,13 @@ #include "core/keyvalue/key_string.h" #include "core/keyvalue/uuid.h" -#include "core/payload/payloadtype.h" +#include "core/namespace/stringsholder.h" #include "cpp-btree/btree_map.h" #include "sparse-map/sparse_map.h" #include "tools/stringstools.h" namespace reindexer { -class FieldsSet; - struct less_key_string { using is_transparent = void; diff --git a/cpp_src/core/index/ttlindex.cc b/cpp_src/core/index/ttlindex.cc index 11e1a8726..7fdbc1e5b 100644 --- a/cpp_src/core/index/ttlindex.cc +++ b/cpp_src/core/index/ttlindex.cc @@ -15,11 +15,13 @@ void UpdateExpireAfter(Index *i, int64_t v) { } } -std::unique_ptr TtlIndex_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr TtlIndex_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { if (idef.opts_.IsPK() || idef.opts_.IsDense()) { - return std::unique_ptr{new TtlIndex>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), + cacheCfg); } - return std::unique_ptr{new TtlIndex>(idef, std::move(payloadType), fields)}; + return std::make_unique>>(idef, std::move(payloadType), std::move(fields), cacheCfg); } } // namespace reindexer diff --git a/cpp_src/core/index/ttlindex.h b/cpp_src/core/index/ttlindex.h index acc0b1a72..b59b6947c 100644 --- a/cpp_src/core/index/ttlindex.h +++ b/cpp_src/core/index/ttlindex.h @@ -10,11 +10,11 @@ namespace reindexer { template class TtlIndex : public IndexOrdered { public: - TtlIndex(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : IndexOrdered(idef, std::move(payloadType), fields), expireAfter_(idef.expireAfter_) {} + TtlIndex(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, const NamespaceCacheConfigData &cacheCfg) + : IndexOrdered(idef, std::move(payloadType), std::move(fields), cacheCfg), expireAfter_(idef.expireAfter_) {} TtlIndex(const TtlIndex &other) : IndexOrdered(other), expireAfter_(other.expireAfter_) {} int64_t GetTTLValue() const noexcept override { return expireAfter_; } - std::unique_ptr Clone() const override { return std::unique_ptr{new TtlIndex(*this)}; } + std::unique_ptr Clone() const override { return std::make_unique>(*this); } void UpdateExpireAfter(int64_t v) noexcept { expireAfter_ = v; } private: @@ -22,7 +22,8 @@ class TtlIndex : public IndexOrdered { int64_t expireAfter_ = 0; }; -std::unique_ptr TtlIndex_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); +std::unique_ptr TtlIndex_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg); void UpdateExpireAfter(Index *i, int64_t v); } // namespace reindexer diff --git a/cpp_src/core/index/uuid_index.cc b/cpp_src/core/index/uuid_index.cc index 57ff52cb4..7622020d3 100644 --- a/cpp_src/core/index/uuid_index.cc +++ b/cpp_src/core/index/uuid_index.cc @@ -10,8 +10,9 @@ void UuidIndex::Upsert(VariantArray &result, const VariantArray &keys, IdType id } } -std::unique_ptr IndexUuid_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { - return std::unique_ptr{new UuidIndex{idef, std::move(payloadType), fields}}; +std::unique_ptr IndexUuid_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields, + const NamespaceCacheConfigData &cacheCfg) { + return std::make_unique(idef, std::move(payloadType), std::move(fields), cacheCfg); } } // namespace reindexer diff --git a/cpp_src/core/index/uuid_index.h b/cpp_src/core/index/uuid_index.h index 1352c3f43..fb2a4411d 100644 --- a/cpp_src/core/index/uuid_index.h +++ b/cpp_src/core/index/uuid_index.h @@ -1,7 +1,6 @@ #pragma once #include "core/index/indexunordered.h" -#include "core/keyvalue/uuid.h" namespace reindexer { @@ -9,12 +8,14 @@ class UuidIndex : public IndexUnordered using Base = IndexUnordered>; public: - UuidIndex(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) : Base{idef, std::move(payloadType), fields} {} - std::unique_ptr Clone() const override { return std::unique_ptr{new UuidIndex{*this}}; } + UuidIndex(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, const NamespaceCacheConfigData& cacheCfg) + : Base{idef, std::move(payloadType), std::move(fields), cacheCfg} {} + std::unique_ptr Clone() const override { return std::make_unique(*this); } using Base::Upsert; void Upsert(VariantArray& result, const VariantArray& keys, IdType id, bool& clearCache) override; // TODO delete this after #1353 }; -std::unique_ptr IndexUuid_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); +std::unique_ptr IndexUuid_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields, + const NamespaceCacheConfigData& cacheCfg); } // namespace reindexer diff --git a/cpp_src/core/indexdef.cc b/cpp_src/core/indexdef.cc index 8195a2fac..9efe90eda 100644 --- a/cpp_src/core/indexdef.cc +++ b/cpp_src/core/indexdef.cc @@ -117,9 +117,9 @@ IndexDef::IndexDef(std::string name, JsonPaths jsonPaths, IndexType type, IndexO this->FromType(type); } -bool IndexDef::IsEqual(const IndexDef &other, bool skipConfig) const { +bool IndexDef::IsEqual(const IndexDef &other, IndexComparison cmpType) const { return name_ == other.name_ && jsonPaths_ == other.jsonPaths_ && Type() == other.Type() && fieldType_ == other.fieldType_ && - opts_.IsEqual(other.opts_, skipConfig) && expireAfter_ == other.expireAfter_; + opts_.IsEqual(other.opts_, cmpType) && expireAfter_ == other.expireAfter_; } IndexType IndexDef::Type() const { diff --git a/cpp_src/core/indexdef.h b/cpp_src/core/indexdef.h index a843a4993..cb4651120 100644 --- a/cpp_src/core/indexdef.h +++ b/cpp_src/core/indexdef.h @@ -26,9 +26,7 @@ struct IndexDef { IndexDef(std::string name, JsonPaths jsonPaths, std::string indexType, std::string fieldType, IndexOpts opts, int64_t expireAfter); IndexDef(std::string name, std::string indexType, std::string fieldType, IndexOpts opts); IndexDef(std::string name, JsonPaths jsonPaths, IndexType type, IndexOpts opts); - bool operator==(const IndexDef &other) const { return IsEqual(other, false); } - bool operator!=(const IndexDef &other) const { return !IsEqual(other, false); } - bool IsEqual(const IndexDef &other, bool skipConfig) const; + bool IsEqual(const IndexDef &other, IndexComparison cmpType) const; IndexType Type() const; std::string getCollateMode() const; const std::vector &Conditions() const; diff --git a/cpp_src/core/indexopts.cc b/cpp_src/core/indexopts.cc index 11948330f..6e8a3cafc 100644 --- a/cpp_src/core/indexopts.cc +++ b/cpp_src/core/indexopts.cc @@ -22,8 +22,9 @@ IndexOpts::IndexOpts(uint8_t flags, CollateMode mode, RTreeIndexType rtreeType) IndexOpts::IndexOpts(const std::string& sortOrderUTF8, uint8_t flags, RTreeIndexType rtreeType) : options(flags), collateOpts_(sortOrderUTF8), rtreeType_(rtreeType) {} -bool IndexOpts::IsEqual(const IndexOpts& other, bool skipConfig) const { - return options == other.options && (skipConfig || config == other.config) && collateOpts_.mode == other.collateOpts_.mode && +bool IndexOpts::IsEqual(const IndexOpts& other, IndexComparison cmpType) const noexcept { + return options == other.options && (cmpType == IndexComparison::SkipConfig || config == other.config) && + collateOpts_.mode == other.collateOpts_.mode && collateOpts_.sortOrderTable.GetSortOrderCharacters() == other.collateOpts_.sortOrderTable.GetSortOrderCharacters() && rtreeType_ == other.rtreeType_; } diff --git a/cpp_src/core/indexopts.h b/cpp_src/core/indexopts.h index 22d86ec92..6b6a8c299 100644 --- a/cpp_src/core/indexopts.h +++ b/cpp_src/core/indexopts.h @@ -12,6 +12,8 @@ struct CollateOpts { void Dump(T& os) const; }; +enum class IndexComparison { WithConfig, SkipConfig }; + /// Cpp version of IndexOpts: includes /// sort order table which is not possible /// to link in C-GO version because of templates @@ -37,8 +39,7 @@ struct IndexOpts { IndexOpts& SetConfig(const std::string& config); CollateMode GetCollateMode() const noexcept; - bool operator==(const IndexOpts& other) const { return IsEqual(other, false); } - bool IsEqual(const IndexOpts& other, bool skipConfig) const; + bool IsEqual(const IndexOpts& other, IndexComparison cmpType) const noexcept; template void Dump(T& os) const; @@ -60,7 +61,6 @@ T& operator<<(T& os, IndexOpts::RTreeIndexType t) { return os << "Greene"; case IndexOpts::RStar: return os << "RStar"; - default: - abort(); } + std::abort(); } diff --git a/cpp_src/core/item.cc b/cpp_src/core/item.cc index 7840e7853..dfbed2b79 100644 --- a/cpp_src/core/item.cc +++ b/cpp_src/core/item.cc @@ -3,7 +3,6 @@ #include "core/itemimpl.h" #include "core/keyvalue/p_string.h" #include "core/namespace/namespace.h" -#include "core/rdxcontext.h" #include "tools/catch_and_return.h" namespace reindexer { @@ -96,7 +95,7 @@ Item::FieldRef &Item::FieldRef::operator=(span arr) { pl.Set(field_, pos++, Variant(elem)); } } else { - if (!itemImpl_->holder_) itemImpl_->holder_.reset(new std::deque); + if (!itemImpl_->holder_) itemImpl_->holder_ = std::make_unique>(); for (auto &elem : arr) { if constexpr (std::is_same_v) { itemImpl_->holder_->push_back(elem.toString()); diff --git a/cpp_src/core/itemimpl.cc b/cpp_src/core/itemimpl.cc index eb8555094..6786925c7 100644 --- a/cpp_src/core/itemimpl.cc +++ b/cpp_src/core/itemimpl.cc @@ -21,7 +21,7 @@ void ItemImpl::SetField(int field, const VariantArray &krs) { !payloadType_.Field(field).Type().Is()) { VariantArray krsCopy; krsCopy.reserve(krs.size()); - if (!holder_) holder_.reset(new std::deque); + if (!holder_) holder_ = std::make_unique>(); for (auto &kr : krs) { holder_->push_back(kr.As()); krsCopy.emplace_back(p_string{&holder_->back()}); @@ -158,10 +158,23 @@ void ItemImpl::FromCJSON(std::string_view slice, bool pkOnly, Recoder *recoder) Serializer rdser(data); Payload pl = GetPayload(); - CJsonDecoder decoder(tagsMatcher_, pkOnly ? &pkFields_ : nullptr, recoder); + if (!holder_) holder_ = std::make_unique>(); + CJsonDecoder decoder(tagsMatcher_, *holder_); + ser_.Reset(); ser_.PutUInt32(0); - decoder.Decode(pl, rdser, ser_); + if (pkOnly && !pkFields_.empty()) { + if rx_unlikely (recoder) { + throw Error(errParams, "ItemImpl::FromCJSON: pkOnly mode is not compatible with non-null recoder"); + } + decoder.Decode(pl, rdser, ser_, CJsonDecoder::RestrictingFilter(pkFields_)); + } else { + if (recoder) { + decoder.Decode(pl, rdser, ser_, CJsonDecoder::DummyFilter(), CJsonDecoder::DefaultRecoder(*recoder)); + } else { + decoder.Decode<>(pl, rdser, ser_); + } + } if (!rdser.Eof()) throw Error(errParseJson, "Internal error - left unparsed data %d", rdser.Pos()); @@ -208,7 +221,7 @@ Error ItemImpl::FromJSON(std::string_view slice, char **endp, bool pkOnly) { } // Split parsed json into indexes and tuple - JsonDecoder decoder(tagsMatcher_, pkOnly ? &pkFields_ : nullptr); + JsonDecoder decoder(tagsMatcher_, pkOnly && !pkFields_.empty() ? &pkFields_ : nullptr); Payload pl = GetPayload(); ser_.Reset(); diff --git a/cpp_src/core/itemimpl.h b/cpp_src/core/itemimpl.h index 2b98aea81..484be7ef1 100644 --- a/cpp_src/core/itemimpl.h +++ b/cpp_src/core/itemimpl.h @@ -98,6 +98,7 @@ class ItemImpl : public ItemImplRawData { std::shared_ptr GetSchema() const noexcept { return schema_; } TagsMatcher &tagsMatcher() noexcept { return tagsMatcher_; } + std::shared_ptr &schema() noexcept { return schema_; } void SetPrecepts(const std::vector &precepts) { precepts_ = precepts; diff --git a/cpp_src/core/itemmodifier.cc b/cpp_src/core/itemmodifier.cc index fb2cfa52c..5f46fc0be 100644 --- a/cpp_src/core/itemmodifier.cc +++ b/cpp_src/core/itemmodifier.cc @@ -1,9 +1,9 @@ #include "itemmodifier.h" +#include "core/itemimpl.h" #include "core/namespace/namespaceimpl.h" #include "core/query/expressionevaluator.h" #include "core/selectfunc/functionexecutor.h" #include "index/index.h" -#include "tools/logger.h" namespace reindexer { diff --git a/cpp_src/core/joincache.h b/cpp_src/core/joincache.h index 8ec4030f2..974373e37 100644 --- a/cpp_src/core/joincache.h +++ b/cpp_src/core/joincache.h @@ -54,14 +54,8 @@ struct JoinCacheVal { bool inited = false; std::shared_ptr preResult; }; -typedef LRUCache MainLruCache; -class JoinCache : public MainLruCache { -public: - JoinCache() : MainLruCache(kDefaultCacheSizeLimit * 2, 2) {} - - typedef std::shared_ptr Ptr; -}; +using JoinCache = LRUCache; struct JoinCacheRes { bool haveData = false; diff --git a/cpp_src/core/key_value_type.h b/cpp_src/core/key_value_type.h index 1f44c8d73..31c58a873 100644 --- a/cpp_src/core/key_value_type.h +++ b/cpp_src/core/key_value_type.h @@ -228,7 +228,7 @@ class KeyValueType { return v.value_ == value_; } [[nodiscard]] bool IsSame(KeyValueType other) const noexcept { return value_ == other.value_; } - [[nodiscard]] TagType ToTagType() const { + [[nodiscard]] TagType ToTagType() const noexcept { switch (value_) { case KVT::Int64: case KVT::Int: diff --git a/cpp_src/core/keyvalue/variant.cc b/cpp_src/core/keyvalue/variant.cc index 71d80b12e..19faf5114 100644 --- a/cpp_src/core/keyvalue/variant.cc +++ b/cpp_src/core/keyvalue/variant.cc @@ -559,8 +559,14 @@ Variant Variant::convert(KeyValueType type, const PayloadType *payloadType, cons Variant &Variant::convert(KeyValueType type, const PayloadType *payloadType, const FieldsSet *fields) & { if (isUuid()) { type.EvaluateOneOf([&](KeyValueType::Uuid) noexcept {}, [&](KeyValueType::String) { *this = Variant{std::string{Uuid{*this}}}; }, + [&](KeyValueType::Composite) { + assertrx_throw(payloadType && fields); + Variant tmp{VariantArray{std::move(*this)}}; + tmp.convertToComposite(*payloadType, *fields); + *this = std::move(tmp); + }, [type](OneOf) { + KeyValueType::Tuple, KeyValueType::Undefined, KeyValueType::Null>) { throw Error(errParams, "Can't convert Variant from type '%s' to type '%s'", KeyValueType{KeyValueType::Uuid{}}.Name(), type.Name()); }); @@ -572,12 +578,22 @@ Variant &Variant::convert(KeyValueType type, const PayloadType *payloadType, con [&](KeyValueType::Int64) { *this = Variant(As()); }, [&](KeyValueType::Double) { *this = Variant(As()); }, [&](KeyValueType::String) { *this = Variant(As()); }, [&](KeyValueType::Composite) { - if (variant_.type.Is()) { - assertrx(payloadType && fields); - convertToComposite(payloadType, fields); - } else { - throw Error(errParams, "Can't convert Variant from type '%s' to type '%s'", variant_.type.Name(), type.Name()); - } + variant_.type.EvaluateOneOf( + [&](KeyValueType::Tuple) { + assertrx(payloadType && fields); + convertToComposite(*payloadType, *fields); + }, + [](KeyValueType::Composite) noexcept {}, + [&](OneOf) { + assertrx(payloadType && fields); + Variant tmp{VariantArray{std::move(*this)}}; + tmp.convertToComposite(*payloadType, *fields); + *this = std::move(tmp); + }, + [&](OneOf) { + throw Error(errParams, "Can't convert Variant from type '%s' to type '%s'", variant_.type.Name(), type.Name()); + }); }, [&](KeyValueType::Uuid) { *this = Variant{As()}; }, [&](OneOf) { @@ -586,7 +602,7 @@ Variant &Variant::convert(KeyValueType type, const PayloadType *payloadType, con return *this; } -void Variant::convertToComposite(const PayloadType *payloadType, const FieldsSet *fields) { +void Variant::convertToComposite(const PayloadType &payloadType, const FieldsSet &fields) { assertrx(!isUuid()); assertrx(variant_.type.Is() && variant_.hold == 1); key_string val = *cast(); @@ -594,24 +610,24 @@ void Variant::convertToComposite(const PayloadType *payloadType, const FieldsSet if (variant_.hold == 1) free(); // Alloc usual payloadvalue + extra memory for hold string - auto &pv = *new (cast()) PayloadValue(payloadType->TotalSize() + val->size()); + auto &pv = *new (cast()) PayloadValue(payloadType.TotalSize() + val->size()); variant_.hold = 1; variant_.type = KeyValueType::Composite{}; // Copy serializer buffer with strings to extra payloadvalue memory - char *data = reinterpret_cast(pv.Ptr() + payloadType->TotalSize()); + char *data = reinterpret_cast(pv.Ptr() + payloadType.TotalSize()); memcpy(data, val->data(), val->size()); Serializer ser(std::string_view(data, val->size())); size_t count = ser.GetVarUint(); - if (count != fields->size()) { - throw Error(errLogic, "Invalid count of arguments for composite index, expected %d, got %d", fields->size(), count); + if (count != fields.size()) { + throw Error(errLogic, "Invalid count of arguments for composite index, expected %d, got %d", fields.size(), count); } - Payload pl(*payloadType, pv); + Payload pl(payloadType, pv); - for (auto field : *fields) { + for (auto field : fields) { if (field != IndexValueType::SetByJsonPath) { pl.Set(field, ser.GetVariant()); } else { diff --git a/cpp_src/core/keyvalue/variant.h b/cpp_src/core/keyvalue/variant.h index a4296aacb..f4d495587 100644 --- a/cpp_src/core/keyvalue/variant.h +++ b/cpp_src/core/keyvalue/variant.h @@ -144,7 +144,7 @@ class Variant { private: bool isUuid() const noexcept { return uuid_.isUuid != 0; } - void convertToComposite(const PayloadType *, const FieldsSet *); + void convertToComposite(const PayloadType &, const FieldsSet &); void free() noexcept; void copy(const Variant &other); template @@ -212,6 +212,8 @@ template <> std::string Variant::As() const; class VariantArray : public h_vector { + using Base = h_vector; + public: VariantArray() noexcept = default; explicit VariantArray(Point) noexcept; @@ -225,9 +227,14 @@ class VariantArray : public h_vector { return std::move(*this); } void MarkObject() noexcept { isObjectValue = true; } - using h_vector::h_vector; - using h_vector::operator==; - using h_vector::operator!=; + using Base::Base; + using Base::operator==; + using Base::operator!=; + template + void clear() noexcept { + isArrayValue = isObjectValue = false; + Base::clear(); + } size_t Hash() const noexcept { size_t ret = this->size(); for (size_t i = 0; i < this->size(); ++i) ret = (ret * 127) ^ this->at(i).Hash(); diff --git a/cpp_src/core/lrucache.cc b/cpp_src/core/lrucache.cc index ecfc29ac8..9ab9316c4 100644 --- a/cpp_src/core/lrucache.cc +++ b/cpp_src/core/lrucache.cc @@ -9,7 +9,7 @@ namespace reindexer { -const int kMaxHitCountToCache = 1024; +constexpr uint32_t kMaxHitCountToCache = 1024; template typename LRUCache::Iterator LRUCache::Get(const K &key) { @@ -29,7 +29,7 @@ typename LRUCache::Iterator LRUCache::Get( it->second.lruPos = std::prev(lru_.end()); } - if (++it->second.hitCount < hitCountToCache_) { + if (++it->second.hitCount < int(hitCountToCache_)) { return Iterator(); } ++getCount_; @@ -57,10 +57,10 @@ void LRUCache::Put(const K &key, V &&v) { eraseLRU(); if rx_unlikely (putCount_ * 16 > getCount_ && eraseCount_) { - logPrintf(LogWarning, "IdSetCache::eraseLRU () cache invalidates too fast eraseCount=%d,putCount=%d,getCount=%d", eraseCount_, - putCount_, eraseCount_); + logPrintf(LogWarning, "IdSetCache::eraseLRU () cache invalidates too fast eraseCount=%d,putCount=%d,getCount=%d,hitCountToCache=%d", + eraseCount_, putCount_, eraseCount_, hitCountToCache_); eraseCount_ = 0; - hitCountToCache_ = std::min(hitCountToCache_ * 2, kMaxHitCountToCache); + hitCountToCache_ = hitCountToCache_ ? std::min(hitCountToCache_ * 2, kMaxHitCountToCache) : 2; putCount_ = 0; getCount_ = 0; } @@ -128,7 +128,7 @@ LRUCacheMemStat LRUCache::GetMemStat() { } template class LRUCache; template class LRUCache; -template class LRUCache; +template class LRUCache; template class LRUCache; } // namespace reindexer diff --git a/cpp_src/core/lrucache.h b/cpp_src/core/lrucache.h index e81fa741f..4cac5b979 100644 --- a/cpp_src/core/lrucache.h +++ b/cpp_src/core/lrucache.h @@ -4,20 +4,18 @@ #include #include #include +#include "dbconfig.h" #include "namespace/namespacestat.h" namespace reindexer { -constexpr size_t kDefaultCacheSizeLimit = 1024 * 1024 * 128; -constexpr int kDefaultHitCountToCache = 2; constexpr size_t kElemSizeOverhead = 256; template class LRUCache { public: using Key = K; - LRUCache(size_t sizeLimit = kDefaultCacheSizeLimit, int hitCount = kDefaultHitCountToCache) noexcept - : totalCacheSize_(0), cacheSizeLimit_(sizeLimit), hitCountToCache_(hitCount) {} + LRUCache(size_t sizeLimit, uint32_t hitCount) noexcept : totalCacheSize_(0), cacheSizeLimit_(sizeLimit), hitCountToCache_(hitCount) {} struct Iterator { Iterator(bool k = false, const V &v = V()) : valid(k), val(v) {} Iterator(const Iterator &other) = delete; @@ -118,7 +116,7 @@ class LRUCache { mutable std::mutex lock_; size_t totalCacheSize_; const size_t cacheSizeLimit_; - int hitCountToCache_; + uint32_t hitCountToCache_; uint64_t getCount_ = 0, putCount_ = 0, eraseCount_ = 0; }; diff --git a/cpp_src/core/namespace/asyncstorage.cc b/cpp_src/core/namespace/asyncstorage.cc index 4f2aa4091..d94d0b08c 100644 --- a/cpp_src/core/namespace/asyncstorage.cc +++ b/cpp_src/core/namespace/asyncstorage.cc @@ -52,7 +52,6 @@ void AsyncStorage::Destroy() { throwOnStorageCopy(); if (storage_) { - tryReopenStorage(); clearUpdates(); storage_->Destroy(path_); reset(); diff --git a/cpp_src/core/namespace/itemsloader.h b/cpp_src/core/namespace/itemsloader.h index 0432a9310..3d39c27c8 100644 --- a/cpp_src/core/namespace/itemsloader.h +++ b/cpp_src/core/namespace/itemsloader.h @@ -1,6 +1,7 @@ #pragma once #include +#include "core/itemimpl.h" #include "namespaceimpl.h" namespace reindexer { diff --git a/cpp_src/core/namespace/namespace.cc b/cpp_src/core/namespace/namespace.cc index 3bc79b243..733bb309e 100644 --- a/cpp_src/core/namespace/namespace.cc +++ b/cpp_src/core/namespace/namespace.cc @@ -35,6 +35,8 @@ void Namespace::CommitTransaction(Transaction& tx, QueryResults& result, const R CounterGuardAIR32 cg(nsl->cancelCommitCnt_); try { auto rlck = statCalculator.CreateLock(*nsl, &NamespaceImpl::rLock, ctx); + tx.ValidatePK(nsl->pkFields()); + auto storageLock = statCalculator.CreateLock(nsl->storage_, &AsyncStorage::FullLock); cg.Reset(); @@ -105,10 +107,14 @@ bool Namespace::needNamespaceCopy(const NamespaceImpl::Ptr& ns, const Transactio void Namespace::doRename(const Namespace::Ptr& dst, const std::string& newName, const std::string& storagePath, const RdxContext& ctx) { std::string dbpath; const auto flushOpts = StorageFlushOpts().WithImmediateReopen(); - awaitMainNs(ctx)->storage_.Flush(flushOpts); auto lck = handleInvalidation(NamespaceImpl::wLock)(ctx); - auto& srcNs = *atomicLoadMainNs(); // -V758 - srcNs.storage_.Flush(flushOpts); // Repeat flush, to raise any disk errors before attempt to close storage + auto srcNsPtr = atomicLoadMainNs(); + auto& srcNs = *srcNsPtr; + srcNs.storage_.Flush(flushOpts); // Repeat flush, to raise any disk errors before attempt to close storage + auto storageStatus = srcNs.storage_.GetStatusCached(); + if (!storageStatus.err.ok()) { + throw Error(storageStatus.err.code(), "Unable to flush storage before rename: %s", storageStatus.err.what()); + } NamespaceImpl::Mutex* dstMtx = nullptr; NamespaceImpl::Ptr dstNs; if (dst) { diff --git a/cpp_src/core/namespace/namespace.h b/cpp_src/core/namespace/namespace.h index 71864740d..55dc59548 100644 --- a/cpp_src/core/namespace/namespace.h +++ b/cpp_src/core/namespace/namespace.h @@ -131,7 +131,12 @@ class Namespace { } void OnConfigUpdated(DBConfigProvider &configProvider, const RdxContext &ctx) { NamespaceConfigData configData; - configProvider.GetNamespaceConfig(GetName(ctx), configData); + const auto nsName = GetName(ctx); + std::string_view realNsName(nsName); + if (isTmpNamespaceNameFast(nsName)) { + realNsName = demangleTmpNamespaceName(realNsName); + } + configProvider.GetNamespaceConfig(realNsName, configData); startCopyPolicyTxSize_.store(configData.startCopyPolicyTxSize, std::memory_order_relaxed); copyPolicyMultiplier_.store(configData.copyPolicyMultiplier, std::memory_order_relaxed); txSizeToAlwaysCopy_.store(configData.txSizeToAlwaysCopy, std::memory_order_relaxed); @@ -211,7 +216,7 @@ class Namespace { ns->tryForceFlush(std::move(locker)); } else if constexpr (std::is_same_v) { auto params = longUpdDelLoggingParams_.load(std::memory_order_relaxed); - const bool isEnabled = params.thresholdUs >= 0 && !isSystemNamespaceNameFast(v._namespace); + const bool isEnabled = params.thresholdUs >= 0 && !isSystemNamespaceNameFast(v.NsName()); auto statCalculator = QueryStatCalculator(long_actions::MakeLogger(v, std::move(params)), isEnabled); auto locker = statCalculator.CreateLock(*ns, &NamespaceImpl::wLock, ctx.rdxContext); calc.LockHit(); diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index 7848af01e..88a448407 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -10,6 +10,7 @@ #include "core/index/ttlindex.h" #include "core/itemimpl.h" #include "core/itemmodifier.h" +#include "core/nsselecter/crashqueryreporter.h" #include "core/nsselecter/nsselecter.h" #include "core/payload/payloadiface.h" #include "core/querystat.h" @@ -72,14 +73,15 @@ NamespaceImpl::NamespaceImpl(const NamespaceImpl& src, AsyncStorage::FullLockT& storage_{src.storage_, storageLock}, replStateUpdates_{src.replStateUpdates_.load()}, meta_{src.meta_}, - queryTotalCountCache_{std::make_shared()}, sparseIndexesCount_{src.sparseIndexesCount_}, krefs{src.krefs}, skrefs{src.skrefs}, sysRecordsVersions_{src.sysRecordsVersions_}, - joinCache_{std::make_shared()}, enablePerfCounters_{src.enablePerfCounters_.load()}, config_{src.config_}, + queryCountCache_{ + std::make_unique(config_.cacheConfig.queryCountCacheSize, config_.cacheConfig.queryCountHitsToCache)}, + joinCache_{std::make_unique(config_.cacheConfig.joinCacheSize, config_.cacheConfig.joinHitsToCache)}, wal_{src.wal_, storage_}, repl_{src.repl_}, observers_{src.observers_}, @@ -99,8 +101,8 @@ NamespaceImpl::NamespaceImpl(const NamespaceImpl& src, AsyncStorage::FullLockT& for (auto& idxIt : src.indexes_) indexes_.push_back(idxIt->Clone()); markUpdated(true); - logPrintf(LogInfo, "Namespace::CopyContentsFrom (%s).Workers: %d, timeout: %d", name_, config_.optimizationSortWorkers, - config_.optimizationTimeout); + logPrintf(LogInfo, "Namespace::CopyContentsFrom (%s).Workers: %d, timeout: %d, tm: { state_token: 0x%08X, version: %d }", name_, + config_.optimizationSortWorkers, config_.optimizationTimeout, tagsMatcher_.stateToken(), tagsMatcher_.version()); } NamespaceImpl::NamespaceImpl(const std::string& name, UpdatesObservers& observers) @@ -109,9 +111,10 @@ NamespaceImpl::NamespaceImpl(const std::string& name, UpdatesObservers& observer name_(name), payloadType_(name), tagsMatcher_(payloadType_), - queryTotalCountCache_(std::make_shared()), - joinCache_(std::make_shared()), enablePerfCounters_(false), + queryCountCache_( + std::make_unique(config_.cacheConfig.queryCountCacheSize, config_.cacheConfig.queryCountHitsToCache)), + joinCache_(std::make_unique(config_.cacheConfig.joinCacheSize, config_.cacheConfig.joinHitsToCache)), wal_(config_.walSize), observers_(&observers), lastSelectTime_{0}, @@ -132,8 +135,8 @@ NamespaceImpl::NamespaceImpl(const std::string& name, UpdatesObservers& observer addIndex(tupleIndexDef); updateSelectTime(); - logPrintf(LogInfo, "Namespace::Construct (%s).Workers: %d, timeout: %d", name_, config_.optimizationSortWorkers, - config_.optimizationTimeout); + logPrintf(LogInfo, "Namespace::Construct (%s).Workers: %d, timeout: %d, tm: { state_token: 0x%08X, version: %d }", name_, + config_.optimizationSortWorkers, config_.optimizationTimeout, tagsMatcher_.stateToken(), tagsMatcher_.version()); } NamespaceImpl::~NamespaceImpl() { @@ -241,6 +244,9 @@ void NamespaceImpl::OnConfigUpdated(DBConfigProvider& configProvider, const RdxC config_.optimizationSortWorkers, configData.optimizationSortWorkers, config_.optimizationTimeout, configData.optimizationTimeout); } + const bool needReconfigureIdxCache = !config_.cacheConfig.IsIndexesCacheEqual(configData.cacheConfig); + const bool needReconfigureJoinCache = !config_.cacheConfig.IsJoinCacheEqual(configData.cacheConfig); + const bool needReconfigureQueryCountCache = !config_.cacheConfig.IsQueryCountCacheEqual(configData.cacheConfig); config_ = configData; storageOpts_.LazyLoad(configData.lazyLoad); storageOpts_.noQueryIdleThresholdSec = configData.noQueryIdleThreshold; @@ -249,6 +255,27 @@ void NamespaceImpl::OnConfigUpdated(DBConfigProvider& configProvider, const RdxC for (auto& idx : indexes_) { idx->EnableUpdatesCountingMode(configData.idxUpdatesCountingMode); } + if (needReconfigureIdxCache) { + for (auto& idx : indexes_) { + idx->ReconfigureCache(config_.cacheConfig); + } + logPrintf(LogTrace, + "[%s] Indexes cache has been reconfigured. IdSets cache (for each index): { max_size %lu KB; hits: %u }. FullTextIdSets " + "cache (for each ft-index): { max_size %lu KB; hits: %u }", + name_, config_.cacheConfig.idxIdsetCacheSize / 1024, config_.cacheConfig.idxIdsetHitsToCache, + config_.cacheConfig.ftIdxCacheSize / 1024, config_.cacheConfig.ftIdxHitsToCache); + } + if (needReconfigureJoinCache) { + joinCache_ = std::make_unique(config_.cacheConfig.joinCacheSize, config_.cacheConfig.joinHitsToCache); + logPrintf(LogTrace, "[%s] Join cache has been reconfigured: { max_size %lu KB; hits: %u }", name_, + config_.cacheConfig.joinCacheSize / 1024, config_.cacheConfig.joinHitsToCache); + } + if (needReconfigureQueryCountCache) { + queryCountCache_ = + std::make_unique(config_.cacheConfig.queryCountCacheSize, config_.cacheConfig.queryCountHitsToCache); + logPrintf(LogTrace, "[%s] Queries count cache has been reconfigured: { max_size %lu KB; hits: %u }", name_, + config_.cacheConfig.queryCountCacheSize / 1024, config_.cacheConfig.queryCountHitsToCache); + } if (needReoptimizeIndexes) { updateSortedIdxCount(); @@ -386,8 +413,8 @@ NamespaceImpl::RollBack_recreateCompositeIndexes NamespaceImpl::re indexDef.opts_ = index->Opts(); indexDef.FromType(index->Type()); - createFieldsSet>(indexDef.name_, index->Type(), index->Fields(), fields); - auto newIndex{Index::New(indexDef, payloadType_, fields)}; + createCompositeFieldsSet>(indexDef.name_, index->Fields(), fields); + auto newIndex{Index::New(indexDef, PayloadType{payloadType_}, FieldsSet{fields}, config_.cacheConfig)}; rollbacker.SaveIndex(std::move(index)); std::swap(index, newIndex); @@ -425,7 +452,7 @@ class NamespaceImpl::RollBack_updateItems : private RollBackBase { } rollbacker_recreateCompositeIndexes_.RollBack(); for (auto& idx : ns_.indexes_) { - idx->UpdatePayloadType(ns_.payloadType_); + idx->UpdatePayloadType(PayloadType{ns_.payloadType_}); } Disable(); } @@ -478,7 +505,7 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con repl_.dataHash, itemsDataSize_}; for (auto& idx : indexes_) { - idx->UpdatePayloadType(payloadType_); + idx->UpdatePayloadType(PayloadType{payloadType_}); } VariantArray skrefsDel, skrefsUps; @@ -588,9 +615,10 @@ void NamespaceImpl::AddIndex(const IndexDef& indexDef, const RdxContext& ctx) { auto wlck = wLock(ctx); - addIndex(indexDef); - saveIndexesToStorage(); - addToWAL(indexDef, WalIndexAdd, ctx); + if (addIndex(indexDef) || ctx.fromReplication_) { + saveIndexesToStorage(); + addToWAL(indexDef, WalIndexAdd, ctx); + } } void NamespaceImpl::DumpIndex(std::ostream& os, std::string_view index, const RdxContext& ctx) const { @@ -600,9 +628,10 @@ void NamespaceImpl::DumpIndex(std::ostream& os, std::string_view index, const Rd void NamespaceImpl::UpdateIndex(const IndexDef& indexDef, const RdxContext& ctx) { auto wlck = wLock(ctx); - updateIndex(indexDef); - saveIndexesToStorage(); - addToWAL(indexDef, WalIndexUpdate, ctx); + if (updateIndex(indexDef) || ctx.fromReplication_) { + saveIndexesToStorage(); + addToWAL(indexDef, WalIndexUpdate, ctx); + } } void NamespaceImpl::DropIndex(const IndexDef& indexDef, const RdxContext& ctx) { @@ -614,6 +643,20 @@ void NamespaceImpl::DropIndex(const IndexDef& indexDef, const RdxContext& ctx) { void NamespaceImpl::SetSchema(std::string_view schema, const RdxContext& ctx) { auto wlck = wLock(ctx); + + if (!ctx.fromReplication_) { + if (schema_ && schema_->GetJSON() == Schema::AppendProtobufNumber(schema, schema_->GetProtobufNsNumber())) { + return; + } + if (repl_.slaveMode) { + logPrintf(LogWarning, + "[repl:%s]:%d Attempt to set new JSON-schema for the replicated namespace via user interface, which does not " + "correspond to the current schema. New schema was ignored to avoid force syncs", + name_, serverId_); + return; + } + } + schema_ = std::make_shared(schema); auto fields = schema_->GetPaths(); for (auto& field : fields) { @@ -745,22 +788,28 @@ void NamespaceImpl::verifyCompositeIndex(const IndexDef& indexDef) const { throw Error{errParams, "Composite index cannot be sparse. Use non-sparse composite instead"}; } for (const auto& jp : indexDef.jsonPaths_) { - const auto it = indexesNames_.find(jp); - if (it == indexesNames_.end()) { + int idx; + if (!getIndexByName(jp, idx)) { if (!IsFullText(indexDef.Type())) { throw Error(errParams, "Composite indexes over non-indexed field ('%s') are not supported yet (except for full-text indexes). Create " "at least column index('-') over each field inside the composite index", jp); } - continue; - } - const auto& idx = indexes_[it->second]; - if (idx->IsUuid() && type != IndexCompositeHash) { - throw Error{errParams, "Only hash index allowed on UUID field"}; - } - if (IsComposite(idx->Type())) { - throw Error(errParams, "Cannot create composite index '%s' over the other composite '%s'", indexDef.name_, idx->Name()); + } else { + const auto& index = *indexes_[idx]; + if (index.Opts().IsSparse()) { + throw Error(errParams, "Composite indexes over sparse indexed field ('%s') are not supported yet", jp); + } + if (type != IndexCompositeHash && index.IsUuid()) { + throw Error{errParams, "Only hash index allowed on UUID field"}; + } + if (index.Opts().IsArray() && !IsFullText(type)) { + throw Error(errParams, "Cannot add array subindex '%s' to not fulltext composite index '%s'", jp, indexDef.name_); + } + if (IsComposite(index.Type())) { + throw Error(errParams, "Cannot create composite index '%s' over the other composite '%s'", indexDef.name_, index.Name()); + } } } } @@ -846,13 +895,13 @@ void NamespaceImpl::verifyUpdateIndex(const IndexDef& indexDef) const { return; } - const auto newIndex = std::unique_ptr(Index::New(indexDef, PayloadType(), FieldsSet())); + const auto newIndex = std::unique_ptr(Index::New(indexDef, PayloadType(), FieldsSet(), config_.cacheConfig)); if (indexDef.opts_.IsSparse()) { if (indexDef.jsonPaths_.size() != 1) { throw Error(errParams, "Sparse index must have exactly 1 JSON-path, but %d paths found for '%s'", indexDef.jsonPaths_.size(), indexDef.name_); } - const auto newSparseIndex = std::unique_ptr(Index::New(indexDef, payloadType_, {})); + const auto newSparseIndex = std::unique_ptr(Index::New(indexDef, PayloadType{payloadType_}, {}, config_.cacheConfig)); } else { FieldsSet changedFields{idxNameIt->second}; PayloadType newPlType = payloadType_; @@ -972,23 +1021,20 @@ class NamespaceImpl::RollBack_addIndex : private RollBackBase { bool needResetPayloadTypeInTagsMatcher_{false}; }; -void NamespaceImpl::addIndex(const IndexDef& indexDef) { +bool NamespaceImpl::addIndex(const IndexDef& indexDef) { const auto& indexName = indexDef.name_; if (const auto idxNameIt = indexesNames_.find(indexName); idxNameIt != indexesNames_.end()) { - IndexDef newIndexDef = indexDef; IndexDef oldIndexDef = getIndexDefinition(indexName); - // reset config - oldIndexDef.opts_.config = ""; - newIndexDef.opts_.config = ""; - if (newIndexDef == oldIndexDef) { - return; + if (indexDef.IsEqual(oldIndexDef, IndexComparison::SkipConfig)) { + return false; } else { if (oldIndexDef.Type() == IndexTtl) { + IndexDef newIndexDef = indexDef; oldIndexDef.expireAfter_ = newIndexDef.expireAfter_; - if (oldIndexDef == newIndexDef) { + if (oldIndexDef.IsEqual(newIndexDef, IndexComparison::SkipConfig)) { auto indx = indexes_[idxNameIt->second].get(); UpdateExpireAfter(indx, newIndexDef.expireAfter_); - return; + return true; } } throw Error(errConflict, "Index '%s.%s' already exists with different settings", name_, indexName); @@ -1005,7 +1051,7 @@ void NamespaceImpl::addIndex(const IndexDef& indexDef) { if (IsComposite(indexDef.Type())) { verifyCompositeIndex(indexDef); addCompositeIndex(indexDef); - return; + return true; } const int idxNo = payloadType_->NumFields(); @@ -1025,20 +1071,20 @@ void NamespaceImpl::addIndex(const IndexDef& indexDef) { TagsPath tagsPath = tagsMatcher_.path2tag(jsonPaths[0], true); assertrx(tagsPath.size() > 0); fields.push_back(std::move(tagsPath)); - auto newIndex = Index::New(indexDef, payloadType_, fields); + auto newIndex = Index::New(indexDef, PayloadType{payloadType_}, std::move(fields), config_.cacheConfig); rollbacker.RollBacker_insertIndex(insertIndex(std::move(newIndex), idxNo, indexName)); ++sparseIndexesCount_; rollbacker.NeedDecreaseSparseIndexCount(); fillSparseIndex(*indexes_[idxNo], jsonPaths[0]); } else { PayloadType oldPlType = payloadType_; - auto newIndex = Index::New(indexDef, PayloadType(), FieldsSet()); + auto newIndex = Index::New(indexDef, PayloadType(), FieldsSet(), config_.cacheConfig); payloadType_.Add(PayloadFieldType{newIndex->KeyType(), indexName, jsonPaths, newIndex->Opts().IsArray()}); rollbacker.SetOldPayloadType(std::move(oldPlType)); tagsMatcher_.UpdatePayloadType(payloadType_); rollbacker.NeedResetPayloadTypeInTagsMatcher(); newIndex->SetFields(FieldsSet(idxNo)); - newIndex->UpdatePayloadType(payloadType_); + newIndex->UpdatePayloadType(PayloadType{payloadType_}); FieldsSet changedFields{0, idxNo}; rollbacker.RollBacker_insertIndex(insertIndex(std::move(newIndex), idxNo, indexName)); @@ -1046,6 +1092,7 @@ void NamespaceImpl::addIndex(const IndexDef& indexDef) { } updateSortedIdxCount(); rollbacker.Disable(); + return true; } void NamespaceImpl::fillSparseIndex(Index& index, std::string_view jsonPath) { @@ -1063,24 +1110,26 @@ void NamespaceImpl::fillSparseIndex(Index& index, std::string_view jsonPath) { markUpdated(false); } -void NamespaceImpl::updateIndex(const IndexDef& indexDef) { +bool NamespaceImpl::updateIndex(const IndexDef& indexDef) { const std::string& indexName = indexDef.name_; IndexDef foundIndex = getIndexDefinition(indexName); - if (indexDef.IsEqual(foundIndex, true)) { + if (indexDef.IsEqual(foundIndex, IndexComparison::SkipConfig)) { // Index has not been changed - if (!indexDef.IsEqual(foundIndex, false)) { + if (!indexDef.IsEqual(foundIndex, IndexComparison::WithConfig)) { // Only index config changed // Just call SetOpts indexes_[getIndexByName(indexName)]->SetOpts(indexDef.opts_); + return true; } - return; + return false; } verifyUpdateIndex(indexDef); dropIndex(indexDef); addIndex(indexDef); + return true; } IndexDef NamespaceImpl::getIndexDefinition(const std::string& indexName) const { @@ -1094,27 +1143,19 @@ IndexDef NamespaceImpl::getIndexDefinition(const std::string& indexName) const { void NamespaceImpl::verifyUpdateCompositeIndex(const IndexDef& indexDef) const { verifyCompositeIndex(indexDef); - IndexType type = indexDef.Type(); - - for (auto& jsonPathOrSubIdx : indexDef.jsonPaths_) { - auto idxNameIt = indexesNames_.find(jsonPathOrSubIdx); - if (idxNameIt != indexesNames_.end() && !indexes_[idxNameIt->second]->Opts().IsSparse() && - indexes_[idxNameIt->second]->Opts().IsArray() && (type == IndexCompositeBTree || type == IndexCompositeHash)) { - throw Error(errParams, "Cannot add array subindex '%s' to composite index '%s'", jsonPathOrSubIdx, indexDef.name_); - } - } - const auto newIndex = std::unique_ptr(Index::New(indexDef, payloadType_, {})); + const auto newIndex = std::unique_ptr(Index::New(indexDef, PayloadType{payloadType_}, {}, config_.cacheConfig)); } void NamespaceImpl::addCompositeIndex(const IndexDef& indexDef) { const auto& indexName = indexDef.name_; FieldsSet fields; - createFieldsSet(indexName, indexDef.Type(), indexDef.jsonPaths_, fields); + createCompositeFieldsSet(indexName, indexDef.jsonPaths_, fields); assertrx(indexesNames_.find(indexName) == indexesNames_.end()); const int idxPos = indexes_.size(); - auto insertIndex_rollbacker{insertIndex(Index::New(indexDef, payloadType_, fields), idxPos, indexName)}; + auto insertIndex_rollbacker{ + insertIndex(Index::New(indexDef, PayloadType{payloadType_}, FieldsSet{fields}, config_.cacheConfig), idxPos, indexName)}; auto indexesCacheCleaner{GetIndexesCacheCleaner()}; for (IdType rowId = 0; rowId < int(items_.size()); rowId++) { @@ -1134,7 +1175,7 @@ void NamespaceImpl::addCompositeIndex(const IndexDef& indexDef) { } template -void NamespaceImpl::createFieldsSet(const std::string& idxName, IndexType type, const PathsT& paths, FieldsSet& fields) { +void NamespaceImpl::createCompositeFieldsSet(const std::string& idxName, const PathsT& paths, FieldsSet& fields) { fields.clear(); const JsonPathsContainerT* jsonPaths = nullptr; @@ -1149,22 +1190,19 @@ void NamespaceImpl::createFieldsSet(const std::string& idxName, IndexType type, } for (const auto& jsonPathOrSubIdx : *jsonPaths) { - auto idxNameIt = indexesNames_.find(jsonPathOrSubIdx); - if (idxNameIt == indexesNames_.end() || idxName == jsonPathOrSubIdx) { + int idx; + if (!getScalarIndexByName(jsonPathOrSubIdx, idx) /* || idxName == jsonPathOrSubIdx*/) { // TODO may be uncomment TagsPath tagsPath = tagsMatcher_.path2tag(jsonPathOrSubIdx, true); if (tagsPath.empty()) { throw Error(errLogic, "Unable to get or create json-path '%s' for composite index '%s'", jsonPathOrSubIdx, idxName); } fields.push_back(tagsPath); fields.push_back(jsonPathOrSubIdx); - } else if (indexes_[idxNameIt->second]->Opts().IsSparse() && !indexes_[idxNameIt->second]->Opts().IsArray()) { - fields.push_back(jsonPathOrSubIdx); - fields.push_back(indexes_[idxNameIt->second]->Fields().getTagsPath(0)); } else { - if (indexes_[idxNameIt->second]->Opts().IsArray() && (type == IndexCompositeBTree || type == IndexCompositeHash)) { - throw Error(errParams, "Cannot add array subindex '%s' to composite index '%s'", jsonPathOrSubIdx, idxName); - } - fields.push_back(idxNameIt->second); + const auto& idxFields = indexes_[idx]->Fields(); + assertrx_throw(idxFields.size() == 1); + assertrx_throw(idxFields[0] >= 0); + fields.push_back(idxFields[0]); } } @@ -1192,6 +1230,16 @@ int NamespaceImpl::getIndexByNameOrJsonPath(std::string_view index) const { } } +int NamespaceImpl::getScalarIndexByName(std::string_view index) const { + int idx; + if (getIndexByName(index, idx)) { + if (idx < indexes_.firstCompositePos()) { + return idx; + } + } + throw Error(errParams, "Index '%s' not found in '%s'", index, name_); +} + bool NamespaceImpl::getIndexByName(std::string_view name, int& index) const { auto it = indexesNames_.find(name); if (it == indexesNames_.end()) return false; @@ -1211,6 +1259,17 @@ bool NamespaceImpl::getIndexByNameOrJsonPath(std::string_view name, int& index) return false; } +bool NamespaceImpl::getScalarIndexByName(std::string_view name, int& index) const { + int idx; + if (getIndexByName(name, idx)) { + if (idx < indexes_.firstCompositePos()) { + index = idx; + return true; + } + } + return false; +} + bool NamespaceImpl::getSparseIndexByJsonPath(std::string_view jsonPath, int& index) const { // FIXME: Try to merge getIndexByNameOrJsonPath and getSparseIndexByJsonPath if it's possible for (int i = indexes_.firstSparsePos(), end = indexes_.firstSparsePos() + indexes_.sparseIndexesSize(); i < end; ++i) { @@ -1234,9 +1293,11 @@ void NamespaceImpl::doUpdate(const Query& query, QueryResults& result, const NsC selCtx.contextCollectingMode = true; selCtx.requiresCrashTracking = true; selCtx.inTransaction = ctx.inTransaction; + selCtx.crashReporterQueryType = QueryUpdate; selecter(result, selCtx, ctx.rdxContext); - auto tmStart = high_resolution_clock::now(); + ActiveQueryScope queryScope(query, QueryUpdate, optimizationState_, strHolder_.get()); + const auto tmStart = high_resolution_clock::now(); bool updateWithJson = false; bool withExpressions = false; @@ -1292,7 +1353,7 @@ void NamespaceImpl::doUpdate(const Query& query, QueryResults& result, const NsC if (!ctx.rdxContext.fromReplication_) setReplLSNs(LSNPair(lsn_t(), lsn)); } - if rx_unlikely (query.debugLevel >= LogInfo) { + if (query.debugLevel >= LogInfo) { logPrintf(LogInfo, "Updated %d items in %d µs", result.Count(), duration_cast(high_resolution_clock::now() - tmStart).count()); } @@ -1345,6 +1406,9 @@ void NamespaceImpl::ModifyItem(Item& item, ItemModifyMode mode, const RdxContext calc.LockHit(); checkApplySlaveUpdate(ctx.fromReplication_); + if (mode == ModeDelete && rx_unlikely(item.PkFields() != pkFields())) { + throw Error(errNotValid, "Item has outdated PK metadata (probably PK has been change during the Delete-call)"); + } modifyItem(item, mode, ctx); tryForceFlush(std::move(wlck)); @@ -1419,12 +1483,15 @@ void NamespaceImpl::doDelete(const Query& q, QueryResults& result, const NsConte selCtx.contextCollectingMode = true; selCtx.requiresCrashTracking = true; selCtx.inTransaction = ctx.inTransaction; + selCtx.crashReporterQueryType = QueryDelete; SelectFunctionsHolder func; selCtx.functions = &func; selecter(result, selCtx, ctx.rdxContext); + + ActiveQueryScope queryScope(q, QueryDelete, optimizationState_, strHolder_.get()); assertrx(result.IsNamespaceAdded(this)); + const auto tmStart = high_resolution_clock::now(); - auto tmStart = high_resolution_clock::now(); AsyncStorage::AdviceGuardT storageAdvice; if (result.Items().size() >= AsyncStorage::kLimitToAdviceBatching) { storageAdvice = storage_.AdviceBatching(); @@ -1442,12 +1509,13 @@ void NamespaceImpl::doDelete(const Query& q, QueryResults& result, const NsConte WrSerializer cjson; for (auto it : result) { cjson.Reset(); - it.GetCJSON(cjson, false); + const auto err = it.GetCJSON(cjson, false); + if (!err.ok()) throw err; const WALRecord wrec{WalItemModify, cjson.Slice(), tagsMatcher_.version(), ModeDelete, ctx.inTransaction}; processWalRecord(wrec, ctx.rdxContext); } } - if rx_unlikely (q.debugLevel >= LogInfo) { + if (q.debugLevel >= LogInfo) { logPrintf(LogInfo, "Deleted %d items in %d µs", result.Count(), duration_cast(high_resolution_clock::now() - tmStart).count()); } @@ -1489,7 +1557,8 @@ void NamespaceImpl::doTruncate(const NsContext& ctx) { itemsDataSize_ = 0; for (size_t i = 0; i < indexes_.size(); ++i) { const IndexOpts opts = indexes_[i]->Opts(); - std::unique_ptr newIdx{Index::New(getIndexDefinition(i), indexes_[i]->GetPayloadType(), indexes_[i]->Fields())}; + std::unique_ptr newIdx{Index::New(getIndexDefinition(i), PayloadType{indexes_[i]->GetPayloadType()}, + FieldsSet{indexes_[i]->Fields()}, config_.cacheConfig)}; newIdx->SetOpts(opts); std::swap(indexes_[i], newIdx); removeIndex(newIdx); @@ -1533,7 +1602,9 @@ void NamespaceImpl::setReplLSNs(LSNPair LSNs) { repl_.originLSN = LSNs.originLSN_; repl_.lastUpstreamLSN = LSNs.upstreamLSN_; replStateUpdates_.fetch_add(1, std::memory_order_release); - logPrintf(LogTrace, "[repl:%s]:%d setReplLSNs originLSN = %s upstreamLSN=%s", name_, serverId_, LSNs.originLSN_, LSNs.upstreamLSN_); + if (!isSystem()) { + logPrintf(LogTrace, "[repl:%s]:%d setReplLSNs originLSN = %s upstreamLSN=%s", name_, serverId_, LSNs.originLSN_, LSNs.upstreamLSN_); + } } void NamespaceImpl::setSlaveMode(const RdxContext& ctx) { @@ -1587,6 +1658,7 @@ void NamespaceImpl::CommitTransaction(Transaction& tx, QueryResults& result, NsC wlck = queryStatCalculator.CreateLock(*this, &NamespaceImpl::wLock, ctx.rdxContext); cg.Reset(); calc.LockHit(); + tx.ValidatePK(pkFields()); } checkApplySlaveUpdate(ctx.rdxContext.fromReplication_); @@ -1751,15 +1823,23 @@ void NamespaceImpl::doUpsert(ItemImpl* ritem, IdType id, bool doUpdate) { Error NamespaceImpl::ReplaceTagsMatcher(const TagsMatcher& tm, const RdxContext& ctx) { CounterGuardAIR32 cg(cancelCommitCnt_); + TagsMatcher oldTm; + auto wlck = wLock(ctx); cg.Reset(); if (items_.size() || !repl_.replicatorEnabled) { return Error(errParams, "Unable to replace tags matcher for %s. Items count: %d, replication flag: %d", name_, items_.size(), repl_.replicatorEnabled); } + oldTm = tagsMatcher_; tagsMatcher_ = tm; tagsMatcher_.UpdatePayloadType(payloadType_); - return errOK; + + logPrintf( + LogInfo, + "[tm:%s]:%d: TagsMatcher was replaced. Old tm: { state_token: 0x%08X, version: %d }, new tm: { state_token: 0x%08X, version: %d }", + name_, serverId_, oldTm.stateToken(), oldTm.version(), tm.stateToken(), tm.version()); + return {}; } void NamespaceImpl::updateTagsMatcherFromItem(ItemImpl* ritem) { @@ -1908,12 +1988,10 @@ void NamespaceImpl::optimizeIndexes(const NsContext& ctx) { rlck = rLock(ctx.rdxContext); } - if (isSystem()) return; - if (!lastUpdateTime || !config_.optimizationTimeout || now - lastUpdateTime < config_.optimizationTimeout) { + if (isSystem() || repl_.temporary || !indexes_.size()) { return; } - - if (!indexes_.size()) { + if (!lastUpdateTime || !config_.optimizationTimeout || now - lastUpdateTime < config_.optimizationTimeout) { return; } @@ -1988,7 +2066,7 @@ void NamespaceImpl::markUpdated(bool forceOptimizeAllIndexes) { int expected{OptimizationCompleted}; optimizationState_.compare_exchange_strong(expected, OptimizedPartially); } - queryTotalCountCache_->Clear(); + queryCountCache_->Clear(); joinCache_->Clear(); lastUpdateTime_.store( std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(), @@ -2005,13 +2083,13 @@ void NamespaceImpl::updateSelectTime() { int64_t NamespaceImpl::getLastSelectTime() const { return lastSelectTime_; } void NamespaceImpl::Select(QueryResults& result, SelectCtx& params, const RdxContext& ctx) { - if (params.query.IsWALQuery()) { + if (!params.query.IsWALQuery()) { + NsSelecter selecter(this); + selecter(result, params, ctx); + } else { WALSelecter selecter(this); selecter(result, params); result.MarkAsWALQuery(); - } else { - NsSelecter selecter(this); - selecter(result, params, ctx); } } @@ -2066,7 +2144,7 @@ NamespaceMemStat NamespaceImpl::GetMemStat(const RdxContext& ctx) { auto rlck = rLock(ctx); ret.name = name_; ret.joinCache = joinCache_->GetMemStat(); - ret.queryCache = queryTotalCountCache_->GetMemStat(); + ret.queryCache = queryCountCache_->GetMemStat(); ret.itemsCount = ItemsCount(); *(static_cast(&ret.replication)) = getReplState(); @@ -2211,6 +2289,8 @@ bool NamespaceImpl::loadIndexesFromStorage() { Serializer ser(def.data(), def.size()); tagsMatcher_.deserialize(ser); tagsMatcher_.clearUpdated(); + logPrintf(LogInfo, "[tm:%s]:%d: TagsMatcher was loaded from storage. tm: { state_token: 0x%08X, version: %d }", name_, serverId_, + tagsMatcher_.stateToken(), tagsMatcher_.version()); logPrintf(LogTrace, "Loaded tags(version: %lld) of namespace %s:\n%s", sysRecordsVersions_.tagsVersion ? sysRecordsVersions_.tagsVersion - 1 : 0, name_, tagsMatcher_.dump()); } @@ -2580,6 +2660,10 @@ Item NamespaceImpl::newItem() { auto impl_ = pool_.get(0, payloadType_, tagsMatcher_, pkFields(), schema_); impl_->tagsMatcher() = tagsMatcher_; impl_->tagsMatcher().clearUpdated(); + impl_->schema() = schema_; +#ifdef RX_WITH_STDLIB_DEBUG + assertrx_dbg(impl_->PkFields() == pkFields()); +#endif // RX_WITH_STDLIB_DEBUG return Item(impl_.release()); } diff --git a/cpp_src/core/namespace/namespaceimpl.h b/cpp_src/core/namespace/namespaceimpl.h index 46fa4d1e1..e7be3ab9e 100644 --- a/cpp_src/core/namespace/namespaceimpl.h +++ b/cpp_src/core/namespace/namespaceimpl.h @@ -20,11 +20,10 @@ #include "core/schema.h" #include "core/storage/idatastorage.h" #include "core/storage/storagetype.h" -#include "core/transactionimpl.h" +#include "core/transaction.h" #include "estl/contexted_locks.h" #include "estl/fast_hash_map.h" #include "estl/shared_mutex.h" -#include "estl/smart_lock.h" #include "estl/syncpool.h" #include "replicator/updatesobserver.h" #include "replicator/waltracker.h" @@ -258,8 +257,10 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. int getIndexByName(std::string_view index) const; int getIndexByNameOrJsonPath(std::string_view name) const; + int getScalarIndexByName(std::string_view name) const; bool getIndexByName(std::string_view name, int &index) const; bool getIndexByNameOrJsonPath(std::string_view name, int &index) const; + bool getScalarIndexByName(std::string_view name, int &index) const; bool getSparseIndexByJsonPath(std::string_view jsonPath, int &index) const; void FillResult(QueryResults &result, const IdSet &ids) const; @@ -281,6 +282,12 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. int getNsNumber() const { return schema_ ? schema_->GetProtobufNsNumber() : 0; } IndexesCacheCleaner GetIndexesCacheCleaner() { return IndexesCacheCleaner{*this}; } void SetDestroyFlag() { dbDestroyed_ = true; } + Error FlushStorage(const RdxContext &ctx) { + const auto flushOpts = StorageFlushOpts().WithImmediateReopen(); + auto lck = rLock(ctx); + storage_.Flush(flushOpts); + return storage_.GetStatusCached().err; + } private: struct SysRecordsVersions { @@ -342,16 +349,16 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. void doDelete(IdType id); void optimizeIndexes(const NsContext &); [[nodiscard]] RollBack_insertIndex insertIndex(std::unique_ptr newIndex, int idxNo, const std::string &realName); - void addIndex(const IndexDef &indexDef); + bool addIndex(const IndexDef &indexDef); void addCompositeIndex(const IndexDef &indexDef); template - void createFieldsSet(const std::string &idxName, IndexType type, const PathsT &paths, FieldsSet &fields); + void createCompositeFieldsSet(const std::string &idxName, const PathsT &paths, FieldsSet &fields); void verifyCompositeIndex(const IndexDef &indexDef) const; template void verifyAddIndex(const IndexDef &indexDef, GetNameF &&) const; void verifyUpdateIndex(const IndexDef &indexDef) const; void verifyUpdateCompositeIndex(const IndexDef &indexDef) const; - void updateIndex(const IndexDef &indexDef); + bool updateIndex(const IndexDef &indexDef); void dropIndex(const IndexDef &index); void addToWAL(const IndexDef &indexDef, WALRecType type, const RdxContext &ctx); void addToWAL(std::string_view json, WALRecType type, const RdxContext &ctx); @@ -413,8 +420,6 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. std::unordered_map meta_; - shared_ptr queryTotalCountCache_; - int sparseIndexesCount_ = 0; VariantArray krefs, skrefs; @@ -431,7 +436,7 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. NamespaceImpl(const NamespaceImpl &src, AsyncStorage::FullLockT &storageLock); - bool isSystem() const { return !name_.empty() && name_[0] == '#'; } + bool isSystem() const { return isSystemNamespaceNameFast(name_); } IdType createItem(size_t realSize); void checkApplySlaveUpdate(bool v); @@ -450,12 +455,12 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. } } - JoinCache::Ptr joinCache_; - PerfStatCounterMT updatePerfCounter_, selectPerfCounter_; std::atomic enablePerfCounters_; NamespaceConfigData config_; + std::unique_ptr queryCountCache_; + std::unique_ptr joinCache_; // Replication variables WALTracker wal_; ReplicationState repl_; diff --git a/cpp_src/core/namespace/namespacestat.h b/cpp_src/core/namespace/namespacestat.h index dd02f8f9b..9fe6ca7bb 100644 --- a/cpp_src/core/namespace/namespacestat.h +++ b/cpp_src/core/namespace/namespacestat.h @@ -22,14 +22,6 @@ struct LRUCacheMemStat { size_t itemsCount = 0; size_t emptyCount = 0; size_t hitCountLimit = 0; - - LRUCacheMemStat &operator+=(const LRUCacheMemStat &other) noexcept { - totalSize += other.totalSize; - itemsCount += other.itemsCount; - emptyCount += other.emptyCount; - hitCountLimit += other.hitCountLimit; - return *this; - } }; struct IndexMemStat { diff --git a/cpp_src/core/nsselecter/aggregator.cc b/cpp_src/core/nsselecter/aggregator.cc index 81194c7d3..837bebab0 100644 --- a/cpp_src/core/nsselecter/aggregator.cc +++ b/cpp_src/core/nsselecter/aggregator.cc @@ -26,8 +26,13 @@ static void copy(It begin, It end, std::vector &facets, const Field ConstPayload pl(payloadType, begin->first); VariantArray va; if (fields[i] == IndexValueType::SetByJsonPath) { - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - pl.GetByJsonPath(tagsPath, va, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + pl.GetByJsonPath(tagsPath, va, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + pl.GetByJsonPath(tagsPath, va, KeyValueType::Undefined{}); + } if (va.IsObjectValue()) { throw Error(errQueryExec, "Cannot aggregate object field"); } @@ -221,7 +226,7 @@ Aggregator::Aggregator(const PayloadType &payloadType, const FieldsSet &fields, } } else { if (sort.empty()) { - facets_ = std::make_unique(MultifieldUnorderedMap{payloadType_, fields_}); + facets_ = std::make_unique(MultifieldUnorderedMap{PayloadType{payloadType_}, FieldsSet{fields_}}); } else { facets_ = std::make_unique(MultifieldOrderedMap{MultifieldComparator{sort, fields_, payloadType_}}); } diff --git a/cpp_src/core/nsselecter/crashqueryreporter.cc b/cpp_src/core/nsselecter/crashqueryreporter.cc index dcfe7036b..7c5fa2ba0 100644 --- a/cpp_src/core/nsselecter/crashqueryreporter.cc +++ b/cpp_src/core/nsselecter/crashqueryreporter.cc @@ -9,36 +9,55 @@ namespace reindexer { struct QueryDebugContext { - SelectCtx *selectCtx = nullptr; + const Query *mainQuery = nullptr; + const Query *parentQuery = nullptr; std::atomic *nsOptimizationState = nullptr; ExplainCalc *explainCalc = nullptr; std::atomic_bool *nsLockerState = nullptr; StringsHolder *nsStrHolder = nullptr; + QueryType realQueryType = QuerySelect; }; thread_local QueryDebugContext g_queryDebugCtx; ActiveQueryScope::ActiveQueryScope(SelectCtx &ctx, std::atomic &nsOptimizationState, ExplainCalc &explainCalc, - std::atomic_bool &nsLockerState, StringsHolder *strHolder) + std::atomic_bool &nsLockerState, StringsHolder *strHolder) noexcept : isTrackedQuery_(ctx.requiresCrashTracking) { if (isTrackedQuery_) { - g_queryDebugCtx.selectCtx = &ctx; + g_queryDebugCtx.mainQuery = &ctx.query; + g_queryDebugCtx.parentQuery = ctx.parentQuery; g_queryDebugCtx.nsOptimizationState = &nsOptimizationState; g_queryDebugCtx.explainCalc = &explainCalc; g_queryDebugCtx.nsLockerState = &nsLockerState; g_queryDebugCtx.nsStrHolder = strHolder; + g_queryDebugCtx.realQueryType = ctx.crashReporterQueryType; } } + +ActiveQueryScope::ActiveQueryScope(const Query &q, QueryType realQueryType, std::atomic &nsOptimizationState, + StringsHolder *strHolder) noexcept + : isTrackedQuery_(true) { + g_queryDebugCtx.mainQuery = &q; + g_queryDebugCtx.parentQuery = nullptr; + g_queryDebugCtx.nsOptimizationState = &nsOptimizationState; + g_queryDebugCtx.explainCalc = nullptr; + g_queryDebugCtx.nsLockerState = nullptr; + g_queryDebugCtx.nsStrHolder = strHolder; + g_queryDebugCtx.realQueryType = realQueryType; +} + ActiveQueryScope::~ActiveQueryScope() { if (isTrackedQuery_) { - if (!g_queryDebugCtx.selectCtx) { - logPrintf(LogWarning, "~ActiveQueryScope: Empty context for tracked query"); + if (!g_queryDebugCtx.mainQuery) { + logPrintf(LogWarning, "~ActiveQueryScope: Empty query pointer in the ActiveQueryScope"); } - g_queryDebugCtx.selectCtx = nullptr; + g_queryDebugCtx.mainQuery = nullptr; + g_queryDebugCtx.parentQuery = nullptr; g_queryDebugCtx.nsOptimizationState = nullptr; g_queryDebugCtx.explainCalc = nullptr; g_queryDebugCtx.nsLockerState = nullptr; g_queryDebugCtx.nsStrHolder = nullptr; + g_queryDebugCtx.realQueryType = QuerySelect; } } @@ -57,41 +76,51 @@ static std::string_view nsOptimizationStateName(int state) { } void PrintCrashedQuery(std::ostream &out) { - if (!g_queryDebugCtx.selectCtx) { + if (!g_queryDebugCtx.mainQuery && !g_queryDebugCtx.parentQuery) { out << "*** No additional info from crash query tracker ***" << std::endl; return; } out << "*** Current query dump ***" << std::endl; - out << " Query: " << g_queryDebugCtx.selectCtx->query.GetSQL() << std::endl; - if (g_queryDebugCtx.selectCtx->parentQuery) { - out << " Parent Query: " << g_queryDebugCtx.selectCtx->parentQuery->GetSQL() << std::endl; + if (g_queryDebugCtx.mainQuery) { + out << " Query: " << g_queryDebugCtx.mainQuery->GetSQL(g_queryDebugCtx.realQueryType) << std::endl; + } + if (g_queryDebugCtx.parentQuery) { + out << " Parent Query: " << g_queryDebugCtx.parentQuery->GetSQL() << std::endl; } - out << " NS state: " << nsOptimizationStateName(g_queryDebugCtx.nsOptimizationState->load()) << std::endl; - out << " NS.locker state: "; - if (g_queryDebugCtx.nsLockerState->load()) { - out << " readonly"; - } else { - out << " regular"; + if (g_queryDebugCtx.nsOptimizationState) { + out << " NS state: " << nsOptimizationStateName(g_queryDebugCtx.nsOptimizationState->load()) << std::endl; } - out << std::endl; - out << " NS.strHolder state: [" << std::endl; - out << " memstat = " << g_queryDebugCtx.nsStrHolder->MemStat() << std::endl; - out << " holds indexes = " << std::boolalpha << g_queryDebugCtx.nsStrHolder->HoldsIndexes() << std::endl; - if (g_queryDebugCtx.nsStrHolder->HoldsIndexes()) { - const auto &indexes = g_queryDebugCtx.nsStrHolder->Indexes(); - out << " indexes.size = " << indexes.size() << std::endl; - out << " indexes = ["; - for (size_t i = 0; i < indexes.size(); ++i) { - if (i) out << " "; - out << indexes[i]->Name(); + if (g_queryDebugCtx.nsLockerState) { + out << " NS.locker state: "; + if (g_queryDebugCtx.nsLockerState->load()) { + out << " readonly"; + } else { + out << " regular"; + } + out << std::endl; + } + if (g_queryDebugCtx.nsStrHolder) { + out << " NS.strHolder state: [" << std::endl; + out << " memstat = " << g_queryDebugCtx.nsStrHolder->MemStat() << std::endl; + out << " holds indexes = " << std::boolalpha << g_queryDebugCtx.nsStrHolder->HoldsIndexes() << std::endl; + if (g_queryDebugCtx.nsStrHolder->HoldsIndexes()) { + const auto &indexes = g_queryDebugCtx.nsStrHolder->Indexes(); + out << " indexes.size = " << indexes.size() << std::endl; + out << " indexes = ["; + for (size_t i = 0; i < indexes.size(); ++i) { + if (i) out << " "; + out << indexes[i]->Name(); + } + out << "]" << std::endl; } out << "]" << std::endl; } - out << "]" << std::endl; - out << " Explain: " << g_queryDebugCtx.explainCalc->GetJSON() << std::endl; + if (g_queryDebugCtx.explainCalc) { + out << " Explain: " << g_queryDebugCtx.explainCalc->GetJSON() << std::endl; + } - g_queryDebugCtx.selectCtx = nullptr; + g_queryDebugCtx.mainQuery = g_queryDebugCtx.parentQuery = nullptr; } } // namespace reindexer diff --git a/cpp_src/core/nsselecter/crashqueryreporter.h b/cpp_src/core/nsselecter/crashqueryreporter.h index 5ed0aff96..b76ab67a2 100644 --- a/cpp_src/core/nsselecter/crashqueryreporter.h +++ b/cpp_src/core/nsselecter/crashqueryreporter.h @@ -2,17 +2,20 @@ #include #include +#include "core/type_consts.h" namespace reindexer { struct SelectCtx; class ExplainCalc; class StringsHolder; +class Query; class ActiveQueryScope { public: ActiveQueryScope(SelectCtx &ctx, std::atomic &nsOptimizationState, ExplainCalc &explainCalc, std::atomic_bool &nsLockerState, - StringsHolder *strHolder); + StringsHolder *strHolder) noexcept; + ActiveQueryScope(const Query &q, QueryType realQueryType, std::atomic &nsOptimizationState, StringsHolder *strHolder) noexcept; ~ActiveQueryScope(); public: diff --git a/cpp_src/core/nsselecter/explaincalc.cc b/cpp_src/core/nsselecter/explaincalc.cc index c7d9506de..484efe308 100644 --- a/cpp_src/core/nsselecter/explaincalc.cc +++ b/cpp_src/core/nsselecter/explaincalc.cc @@ -45,7 +45,7 @@ void ExplainCalc::LogDump(int logLevel) { } } -constexpr inline const char *joinTypeName(JoinType type) noexcept { +constexpr static inline const char *joinTypeName(JoinType type) noexcept { switch (type) { case JoinType::InnerJoin: return "inner_join "; @@ -60,7 +60,7 @@ constexpr inline const char *joinTypeName(JoinType type) noexcept { } } -constexpr inline const char *opName(OpType op, bool first = true) { +constexpr static inline const char *opName(OpType op, bool first = true) { switch (op) { case OpAnd: return first ? "" : "and "; diff --git a/cpp_src/core/nsselecter/fieldscomparator.h b/cpp_src/core/nsselecter/fieldscomparator.h index dfe375bd0..cda7dd1ff 100644 --- a/cpp_src/core/nsselecter/fieldscomparator.h +++ b/cpp_src/core/nsselecter/fieldscomparator.h @@ -18,13 +18,13 @@ class FieldsComparator { const std::string& Name() const&& = delete; std::string Dump() const { return Name(); } int GetMatchedCount() const noexcept { return matchedCount_; } - void SetLeftField(const TagsPath& tpath) { - setField(tpath, ctx_[0].lCtx_); + void SetLeftField(const FieldsSet& fields) { + setField(fields, ctx_[0].lCtx_); leftFieldSet = true; } - void SetRightField(const TagsPath& tpath) { + void SetRightField(const FieldsSet& fields) { assertrx(leftFieldSet); - setField(tpath, ctx_[0].rCtx_); + setField(fields, ctx_[0].rCtx_); } void SetLeftField(const FieldsSet& fset, KeyValueType type, bool isArray) { if (type.Is()) { @@ -67,6 +67,11 @@ class FieldsComparator { }; void setField(const TagsPath& tpath, FieldContext& fctx) { fctx.fields_.push_back(tpath); } + void setField(const FieldsSet& fields, FieldContext& fctx) { + assertrx_throw(fields.size() == 1); + assertrx_throw(fields[0] == IndexValueType::SetByJsonPath); + setField(fields.getTagsPath(0), fctx); + } void setField(FieldContext& fctx, FieldsSet fset, KeyValueType type, bool isArray) { fctx.fields_ = std::move(fset); fctx.type_ = type; diff --git a/cpp_src/core/nsselecter/joinedselector.cc b/cpp_src/core/nsselecter/joinedselector.cc index 84fbe4b8b..7492f23a5 100644 --- a/cpp_src/core/nsselecter/joinedselector.cc +++ b/cpp_src/core/nsselecter/joinedselector.cc @@ -3,6 +3,7 @@ #include "core/namespace/namespaceimpl.h" #include "core/queryresults/joinresults.h" #include "nsselecter.h" +#include "vendor/sparse-map/sparse_set.h" constexpr size_t kMaxIterationsScaleForInnerJoinOptimization = 100; @@ -53,8 +54,8 @@ void JoinedSelector::selectFromPreResultValues(QueryResults &joinItemR, const Qu for (const ItemRef &item : preResult_->values) { auto &v = item.Value(); assertrx(!v.IsFree()); - if (query.entries.CheckIfSatisfyConditions({preResult_->values.payloadType, v}, preResult_->values.tagsMatcher)) { - if (++matched > query.count) break; + if (query.entries.CheckIfSatisfyConditions({preResult_->values.payloadType, v})) { + if (++matched > query.Limit()) break; found = true; joinItemR.Add(item); } @@ -76,21 +77,12 @@ bool JoinedSelector::Process(IdType rowId, int nsId, ConstPayload payload, bool std::unique_ptr itemQueryCopy; Query *itemQueryPtr = &itemQuery_; for (auto &je : joinQuery_.joinEntries_) { - const bool nonIndexedField = (je.idxNo == IndexValueType::SetByJsonPath); - if (nonIndexedField) { - VariantArray &values = itemQueryPtr->entries.Get(i).values; - const KeyValueType type{values.empty() ? KeyValueType::Undefined{} : values[0].Type()}; - payload.GetByJsonPath(je.index_, leftNs_->tagsMatcher_, values, type); - } else { - const auto &index = *leftNs_->indexes_[je.idxNo]; - const auto &fields = index.Fields(); - if (fields.getJsonPathsLength() == 0) { - payload.Get(fields[0], itemQueryPtr->entries.Get(i).values); - } else { - payload.GetByJsonPath(fields.getTagsPath(0), itemQueryPtr->entries.Get(i).values, index.KeyType()); - } + QueryEntry &qentry = itemQueryPtr->entries.Get(i); + { + auto keyValues = qentry.UpdatableValues(QueryEntry::IgnoreEmptyValues{}); + payload.GetByFieldsSet(je.LeftFields(), keyValues, je.LeftFieldType(), je.LeftCompositeFieldsTypes()); } - if (itemQueryPtr->entries.Get(i).values.empty()) { + if (qentry.Values().empty()) { if (itemQueryPtr == &itemQuery_) { itemQueryCopy = std::unique_ptr{new Query(itemQuery_)}; itemQueryPtr = itemQueryCopy.get(); @@ -99,7 +91,7 @@ bool JoinedSelector::Process(IdType rowId, int nsId, ConstPayload payload, bool } ++i; } - itemQueryPtr->Limit(match ? joinQuery_.count : 0); + itemQueryPtr->Limit(match ? joinQuery_.Limit() : 0); bool found = false; bool matchedAtLeastOnce = false; @@ -122,58 +114,70 @@ bool JoinedSelector::Process(IdType rowId, int nsId, ConstPayload payload, bool return matchedAtLeastOnce; } -template -void JoinedSelector::readValuesFromRightNs(VariantArray &values, const KeyValueType leftIndexType, [[maybe_unused]] int rightIdxNo, - [[maybe_unused]] std::string_view rightIndex) const { - std::unordered_set set; - VariantArray buffer; - for (IdType rowId : preResult_->ids) { - if (rightNs_->items_[rowId].IsFree()) continue; - buffer.clear(); - const ConstPayload pl{rightNs_->payloadType_, rightNs_->items_[rowId]}; - if constexpr (byJsonPath) { - pl.GetByJsonPath(rightIndex, rightNs_->tagsMatcher_, buffer, leftIndexType); - } else { - pl.Get(rightIdxNo, buffer); +template +VariantArray JoinedSelector::readValuesOfRightNsFrom(const Cont &data, const Fn &createPayload, const QueryJoinEntry &entry, + const PayloadType &pt) const { + const auto rightFieldType = entry.RightFieldType(); + const auto leftFieldType = entry.LeftFieldType(); + VariantArray res; + if (rightFieldType.Is()) { + unordered_payload_set set(data.size(), hash_composite(pt, entry.RightFields()), equal_composite(pt, entry.RightFields())); + for (const auto &v : data) { + const auto pl = createPayload(v); + if (pl) { + set.insert(*pl->Value()); + } } - if (!leftIndexType.Is() && !leftIndexType.Is()) { - for (Variant &v : buffer) set.insert(std::move(v.convert(leftIndexType))); - } else { - for (Variant &v : buffer) set.insert(std::move(v)); + res.reserve(set.size()); + for (auto &s : set) { + res.emplace_back(std::move(s)); } - } - values.reserve(set.size()); - std::move(set.begin(), set.end(), std::back_inserter(values)); -} - -template -void JoinedSelector::readValuesFromPreResult(VariantArray &values, const KeyValueType leftIndexType, int rightIdxNo, - std::string_view rightIndex) const { - std::unordered_set set; - VariantArray buffer; - for (const ItemRef &item : preResult_->values) { - buffer.clear(); - assertrx(!item.Value().IsFree()); - const ConstPayload pl{preResult_->values.payloadType, item.Value()}; - if constexpr (byJsonPath) { - pl.GetByJsonPath(rightIndex, preResult_->values.tagsMatcher, buffer, leftIndexType); - (void)rightIdxNo; - } else { - pl.Get(rightIdxNo, buffer); - (void)rightIndex; + } else { + tsl::sparse_set set(data.size()); + for (const auto &v : data) { + const auto pl = createPayload(v); + if (!pl) { + continue; + } + pl->GetByFieldsSet(entry.RightFields(), res, entry.RightFieldType(), entry.RightCompositeFieldsTypes()); + if (!leftFieldType.Is() && !leftFieldType.Is()) { + for (Variant &v : res) set.insert(std::move(v.convert(leftFieldType))); + } else { + for (Variant &v : res) set.insert(std::move(v)); + } } - if (!leftIndexType.Is() && !leftIndexType.Is()) { - for (Variant &v : buffer) set.insert(std::move(v.convert(leftIndexType))); - } else { - for (Variant &v : buffer) set.insert(std::move(v)); + res.clear(); + for (auto &s : set) { + res.emplace_back(std::move(s)); } } - values.reserve(set.size()); - std::move(set.begin(), set.end(), std::back_inserter(values)); + return res; +} + +VariantArray JoinedSelector::readValuesFromRightNs(const QueryJoinEntry &entry) const { + return readValuesOfRightNsFrom( + preResult_->ids, + [this](IdType rowId) -> std::optional { + const auto &item = rightNs_->items_[rowId]; + if (item.IsFree()) { + return std::nullopt; + } + return ConstPayload{rightNs_->payloadType_, item}; + }, + entry, rightNs_->payloadType_); } -template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; -template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; +VariantArray JoinedSelector::readValuesFromPreResult(const QueryJoinEntry &entry) const { + return readValuesOfRightNsFrom( + preResult_->values, + [this](const ItemRef &item) -> std::optional { + if (item.Value().IsFree()) { + return std::nullopt; + } + return ConstPayload{preResult_->values.payloadType, item.Value()}; + }, + entry, preResult_->values.payloadType); +} void JoinedSelector::AppendSelectIteratorOfJoinIndexData(SelectIteratorContainer &iterators, int *maxIterations, unsigned sortId, const SelectFunction::Ptr &selectFnc, const RdxContext &rdxCtx) { @@ -184,39 +188,25 @@ void JoinedSelector::AppendSelectIteratorOfJoinIndexData(SelectIteratorContainer return; } unsigned optimized = 0; - assertrx(preResult_->dataMode != JoinPreResult::ModeValues || itemQuery_.entries.Size() == joinQuery_.joinEntries_.size()); + assertrx_throw(preResult_->dataMode != JoinPreResult::ModeValues || itemQuery_.entries.Size() == joinQuery_.joinEntries_.size()); for (size_t i = 0; i < joinQuery_.joinEntries_.size(); ++i) { const QueryJoinEntry &joinEntry = joinQuery_.joinEntries_[i]; - if (joinEntry.op_ != OpAnd || (joinEntry.condition_ != CondEq && joinEntry.condition_ != CondSet) || - (i + 1 < joinQuery_.joinEntries_.size() && joinQuery_.joinEntries_[i + 1].op_ == OpOr) || - joinEntry.idxNo == IndexValueType::SetByJsonPath) { + if (!joinEntry.IsLeftFieldIndexed() || joinEntry.Operation() != OpAnd || + (joinEntry.Condition() != CondEq && joinEntry.Condition() != CondSet) || + (i + 1 < joinQuery_.joinEntries_.size() && joinQuery_.joinEntries_[i + 1].Operation() == OpOr)) { continue; } - const auto &leftIndex = leftNs_->indexes_[joinEntry.idxNo]; + const auto &leftIndex = leftNs_->indexes_[joinEntry.LeftIdxNo()]; assertrx(!IsFullText(leftIndex->Type())); if (leftIndex->Opts().IsSparse()) continue; VariantArray values; if (preResult_->dataMode == JoinPreResult::ModeIdSet) { - int rightIdxNo = IndexValueType::NotSet; - if (rightNs_->getIndexByNameOrJsonPath(joinEntry.joinIndex_, rightIdxNo) && - !rightNs_->indexes_[rightIdxNo]->Opts().IsSparse()) { - readValuesFromRightNs(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } else { - readValuesFromRightNs(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } + values = readValuesFromRightNs(joinEntry); } else { - assertrx(itemQuery_.entries.HoldsOrReferTo(i)); - const QueryEntry &qe = itemQuery_.entries.Get(i); - assertrx(qe.index == joinEntry.joinIndex_); - const int rightIdxNo = qe.idxNo; - if (rightIdxNo == IndexValueType::SetByJsonPath) { - readValuesFromPreResult(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } else { - readValuesFromPreResult(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } + values = readValuesFromPreResult(joinEntry); } - auto ctx = selectFnc ? selectFnc->CreateCtx(joinEntry.idxNo) : BaseFunctionCtx::Ptr{}; + auto ctx = selectFnc ? selectFnc->CreateCtx(joinEntry.LeftIdxNo()) : BaseFunctionCtx::Ptr{}; assertrx(!ctx || ctx->type != BaseFunctionCtx::kFtCtx); if (leftIndex->Opts().GetCollateMode() == CollateUTF8) { @@ -230,9 +220,9 @@ void JoinedSelector::AppendSelectIteratorOfJoinIndexData(SelectIteratorContainer bool was = false; for (SelectKeyResult &res : leftIndex->SelectKey(values, CondSet, sortId, opts, ctx, rdxCtx)) { if (!res.comparators_.empty()) continue; - SelectIterator selIter{res, false, joinEntry.index_, - (joinEntry.idxNo < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed), false}; - selIter.Bind(leftNs_->payloadType_, joinEntry.idxNo); + SelectIterator selIter{res, false, joinEntry.LeftFieldName(), + (joinEntry.LeftIdxNo() < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed), false}; + selIter.Bind(leftNs_->payloadType_, joinEntry.LeftIdxNo()); const int curIterations = selIter.GetMaxIterations(); if (curIterations && curIterations < *maxIterations) *maxIterations = curIterations; iterators.Append(OpAnd, std::move(selIter)); diff --git a/cpp_src/core/nsselecter/joinedselector.h b/cpp_src/core/nsselecter/joinedselector.h index 0fb63758d..531a1c25e 100644 --- a/cpp_src/core/nsselecter/joinedselector.h +++ b/cpp_src/core/nsselecter/joinedselector.h @@ -88,7 +88,13 @@ class JoinedSelector { joinedSelectorsCount_(joinedSelectorsCount), rdxCtx_(rdxCtx), optimized_(false), - inTransaction_{inTransaction} {} + inTransaction_{inTransaction} { +#ifndef NDEBUG + for (const auto &jqe : joinQuery_.joinEntries_) { + assertrx_throw(jqe.FieldsHaveBeenSet()); + } +#endif + } JoinedSelector(JoinedSelector &&) = default; JoinedSelector &operator=(JoinedSelector &&) = delete; @@ -98,7 +104,7 @@ class JoinedSelector { bool Process(IdType, int nsId, ConstPayload, bool match); JoinType Type() const noexcept { return joinType_; } void SetType(JoinType type) noexcept { joinType_ = type; } - const std::string &RightNsName() const noexcept { return itemQuery_._namespace; } + const std::string &RightNsName() const noexcept { return itemQuery_.NsName(); } const JoinedQuery &JoinQuery() const noexcept { return joinQuery_; } int Called() const noexcept { return called_; } int Matched() const noexcept { return matched_; } @@ -109,10 +115,11 @@ class JoinedSelector { const NamespaceImpl::Ptr &RightNs() const noexcept { return rightNs_; } private: - template - void readValuesFromRightNs(VariantArray &values, KeyValueType leftIndexType, int rightIdxNo, std::string_view rightIndex) const; - template - void readValuesFromPreResult(VariantArray &values, KeyValueType leftIndexType, int rightIdxNo, std::string_view rightIndex) const; + [[nodiscard]] VariantArray readValuesFromRightNs(const QueryJoinEntry &) const; + [[nodiscard]] VariantArray readValuesFromPreResult(const QueryJoinEntry &) const; + template + [[nodiscard]] VariantArray readValuesOfRightNsFrom(const Cont &from, const Fn &createPayload, const QueryJoinEntry &, + const PayloadType &) const; void selectFromRightNs(QueryResults &joinItemR, const Query &, bool &found, bool &matchedAtLeastOnce); void selectFromPreResultValues(QueryResults &joinItemR, const Query &, bool &found, bool &matchedAtLeastOnce) const; @@ -134,7 +141,4 @@ class JoinedSelector { }; using JoinedSelectors = std::vector; -extern template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; -extern template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; - } // namespace reindexer diff --git a/cpp_src/core/nsselecter/joinedselectormock.h b/cpp_src/core/nsselecter/joinedselectormock.h index 5db3a49e6..e5cc58ea3 100644 --- a/cpp_src/core/nsselecter/joinedselectormock.h +++ b/cpp_src/core/nsselecter/joinedselectormock.h @@ -17,7 +17,7 @@ class JoinedSelectorMock { public: JoinedSelectorMock(JoinType jt, reindexer::JoinedQuery q) : query_{std::move(q)}, qr_{}, joinType_{jt} {} const reindexer::JoinedQuery& JoinQuery() const noexcept { return query_; } - const std::string& RightNsName() const noexcept { return query_._namespace; } + const std::string& RightNsName() const noexcept { return query_.NsName(); } reindexer::QueryResults& QueryResults() noexcept { return qr_; } const reindexer::QueryResults& QueryResults() const noexcept { return qr_; } JoinType Type() const noexcept { return joinType_; } diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index 0e7bf7650..937707fb9 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -1,6 +1,5 @@ #include "nsselecter.h" -#include "core/cjson/jsonbuilder.h" #include "core/namespace/namespaceimpl.h" #include "core/queryresults/joinresults.h" #include "crashqueryreporter.h" @@ -48,13 +47,13 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte bool containAggCount = containSomeAggCount(AggCount); bool containAggCountCached = containAggCount ? false : containSomeAggCount(AggCountCached); - bool needCalcTotal = aggregationQueryRef.calcTotal == ModeAccurateTotal || containAggCount; + bool needCalcTotal = aggregationQueryRef.CalcTotal() == ModeAccurateTotal || containAggCount; QueryCacheKey ckey; - if (aggregationQueryRef.calcTotal == ModeCachedTotal || containAggCountCached) { + if (aggregationQueryRef.CalcTotal() == ModeCachedTotal || containAggCountCached) { ckey = QueryCacheKey{ctx.query}; - auto cached = ns_->queryTotalCountCache_->Get(ckey); + auto cached = ns_->queryCountCache_->Get(ckey); if (cached.valid && cached.val.total_count >= 0) { result.totalCount += cached.val.total_count; logPrintf(LogTrace, "[%s] using value from cache: %d", ns_->name_, result.totalCount); @@ -92,7 +91,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte } // Prepare data for select functions if (ctx.functions) { - fnc_ = ctx.functions->AddNamespace(ctx.query, *ns_, isFt); + fnc_ = ctx.functions->AddNamespace(ctx.query, *ns_, ctx.nsid, isFt); } if (!ctx.skipIndexesLookup) { @@ -362,10 +361,10 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte } } // Put count/count_cached to aggretions - if (aggregationQueryRef.calcTotal != ModeNoTotal || containAggCount || containAggCountCached) { + if (aggregationQueryRef.CalcTotal() != ModeNoTotal || containAggCount || containAggCountCached) { AggregationResult ret; ret.fields = {"*"}; - ret.type = (aggregationQueryRef.calcTotal == ModeAccurateTotal || containAggCount) ? AggCount : AggCountCached; + ret.type = (aggregationQueryRef.CalcTotal() == ModeAccurateTotal || containAggCount) ? AggCount : AggCountCached; if (ctx.isMergeQuerySubQuery()) { assertrx_throw(!result.aggregationResults.empty()); auto &agg = result.aggregationResults.back(); @@ -404,7 +403,7 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte if (needPutCachedTotal) { logPrintf(LogTrace, "[%s] put totalCount value into query cache: %d ", ns_->name_, result.totalCount); - ns_->queryTotalCountCache_->Put(ckey, {static_cast(result.totalCount - initTotalCount)}); + ns_->queryCountCache_->Put(ckey, {static_cast(result.totalCount - initTotalCount)}); } if (ctx.preResult && ctx.preResult->executionMode == JoinPreResult::ModeBuild) { switch (ctx.preResult->dataMode) { @@ -783,7 +782,7 @@ It NsSelecter::applyForcedSortImpl(NamespaceImpl &ns, It begin, It end, const It // implementation for composite indexes const auto &payloadType = ns.payloadType_; const FieldsSet &fields = ns.indexes_[idx]->Fields(); - unordered_payload_map sortMap(0, payloadType, fields); + unordered_payload_map sortMap(0, PayloadType{payloadType}, FieldsSet{fields}); ForcedMapInserter inserter{sortMap}; for (auto value : forcedSortOrder) { value.convert(fieldType, &payloadType, &fields); @@ -1323,7 +1322,7 @@ void NsSelecter::prepareSortJoinedIndex(size_t nsIdx, std::string_view column, i .FieldByName(std::string{column}, index); if (index == IndexValueType::SetByJsonPath) { skipSortingEntry |= !validateField( - strictMode, column, js.joinQuery_._namespace, + strictMode, column, js.joinQuery_.NsName(), js.preResult_->dataMode == JoinPreResult::ModeValues ? js.preResult_->values.tagsMatcher : js.rightNs_->tagsMatcher_); } } @@ -1565,11 +1564,11 @@ size_t NsSelecter::calculateNormalCost(const QueryEntries &qentries, SelectCtx & [&costCalculator](const JoinQueryEntry &) { costCalculator.MarkInapposite(); }, [&costCalculator](const BetweenFieldsQueryEntry &) { costCalculator.MarkInapposite(); }, [&](const QueryEntry &qe) { - if (qe.idxNo < 0) { + if (!qe.IsFieldIndexed()) { costCalculator.MarkInapposite(); return; } - if (qe.idxNo == ctx.sortingContext.uncommitedIndex) { + if (qe.IndexNo() == ctx.sortingContext.uncommitedIndex) { if (sortIndexSearchState == SortIndexNotFound) { const bool isExpectingIdSet = qentries.GetOperation(i) == OpAnd && (next == sz || qentries.GetOperation(next) != OpOr); @@ -1591,7 +1590,7 @@ size_t NsSelecter::calculateNormalCost(const QueryEntries &qentries, SelectCtx & return; } - auto &index = ns_->indexes_[qe.idxNo]; + auto &index = ns_->indexes_[qe.IndexNo()]; if (IsFullText(index->Type())) { costCalculator.MarkInapposite(); return; @@ -1604,8 +1603,8 @@ size_t NsSelecter::calculateNormalCost(const QueryEntries &qentries, SelectCtx & opts.inTransaction = ctx.inTransaction; try { - SelectKeyResults reslts = index->SelectKey(qe.values, qe.condition, 0, opts, nullptr, rdxCtx); - costCalculator.Add(reslts, qe.idxNo == ctx.sortingContext.uncommitedIndex); + SelectKeyResults reslts = index->SelectKey(qe.Values(), qe.Condition(), 0, opts, nullptr, rdxCtx); + costCalculator.Add(reslts, qe.IndexNo() == ctx.sortingContext.uncommitedIndex); } catch (const Error &) { costCalculator.MarkInapposite(); } @@ -1632,7 +1631,7 @@ size_t NsSelecter::calculateOptimizedCost(size_t costNormal, const QueryEntries [&costCalculator](const JoinQueryEntry &) { costCalculator.MarkInapposite(); }, [&costCalculator](const BetweenFieldsQueryEntry &) { costCalculator.MarkInapposite(); }, [&](const QueryEntry &qe) { - if (qe.idxNo < 0 || qe.idxNo != ctx.sortingContext.uncommitedIndex) { + if (!qe.IsFieldIndexed() || qe.IndexNo() != ctx.sortingContext.uncommitedIndex) { costCalculator.MarkInapposite(); return; } @@ -1645,7 +1644,7 @@ size_t NsSelecter::calculateOptimizedCost(size_t costNormal, const QueryEntries opts.inTransaction = ctx.inTransaction; try { - SelectKeyResults reslts = ns_->indexes_[qe.idxNo]->SelectKey(qe.values, qe.condition, 0, opts, nullptr, rdxCtx); + SelectKeyResults reslts = ns_->indexes_[qe.IndexNo()]->SelectKey(qe.Values(), qe.Condition(), 0, opts, nullptr, rdxCtx); costCalculator.Add(reslts); } catch (const Error &) { costCalculator.MarkInapposite(); @@ -1662,7 +1661,7 @@ bool NsSelecter::isSortOptimizatonEffective(const QueryEntries &qentries, Select } if (qentries.Size() == 1 && qentries.HoldsOrReferTo(0)) { const auto &qe = qentries.Get(0); - if (qe.idxNo == ctx.sortingContext.uncommitedIndex) { + if (qe.IndexNo() == ctx.sortingContext.uncommitedIndex) { return SelectIteratorContainer::IsExpectingOrderedResults(qe); } } @@ -1696,8 +1695,8 @@ bool NsSelecter::isSortOptimizatonEffective(const QueryEntries &qentries, Select // TODO: It's possible to evaluate this multiplier, based on the query conditions, but the only way to avoid corner cases is to // allow user to hint this optimization. const size_t limitMultiplier = std::max(size_t(20), size_t(totalItemsCount / expectedMaxIterationsNormal) * 4); - const auto offset = ctx.query.HasOffset() ? ctx.query.start : 1; - costOptimized = limitMultiplier * (ctx.query.count + offset); + const auto offset = ctx.query.HasOffset() ? ctx.query.Offset() : 1; + costOptimized = limitMultiplier * (ctx.query.Limit() + offset); } return costOptimized <= costNormal; } diff --git a/cpp_src/core/nsselecter/nsselecter.h b/cpp_src/core/nsselecter/nsselecter.h index 1a51ab7eb..6d4c87f34 100644 --- a/cpp_src/core/nsselecter/nsselecter.h +++ b/cpp_src/core/nsselecter/nsselecter.h @@ -11,7 +11,7 @@ enum class IsMergeQuery : bool { Yes = true, No = false }; enum class IsFTQuery { Yes, No, NotSet }; struct SelectCtx { - explicit SelectCtx(const Query &query_, const Query *parentQuery_) : query(query_), parentQuery(parentQuery_) {} + explicit SelectCtx(const Query &query_, const Query *parentQuery_) noexcept : query(query_), parentQuery(parentQuery_) {} const Query &query; JoinedSelectors *joinedSelectors = nullptr; SelectFunctionsHolder *functions = nullptr; @@ -28,6 +28,7 @@ struct SelectCtx { bool inTransaction = false; IsMergeQuery isMergeQuery = IsMergeQuery::No; IsFTQuery isFtQuery = IsFTQuery::NotSet; + QueryType crashReporterQueryType = QuerySelect; const Query *parentQuery = nullptr; ExplainCalc explain; @@ -80,7 +81,7 @@ class NsSelecter { void addSelectResult(uint8_t proc, IdType rowId, IdType properRowId, SelectCtx &sctx, h_vector &aggregators, QueryResults &result, bool preselectForFt); - h_vector getAggregators(const std::vector& aggEntrys, StrictMode strictMode) const; + h_vector getAggregators(const std::vector &aggEntrys, StrictMode strictMode) const; void setLimitAndOffset(ItemRefVector &result, size_t offset, size_t limit); void prepareSortingContext(SortingEntries &sortBy, SelectCtx &ctx, bool isFt, bool availableSelectBySortIndex); static void prepareSortIndex(const NamespaceImpl &, std::string_view column, int &index, bool &skipSortingEntry, StrictMode); diff --git a/cpp_src/core/nsselecter/querypreprocessor.cc b/cpp_src/core/nsselecter/querypreprocessor.cc index e7abaf49e..6070eb478 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.cc +++ b/cpp_src/core/nsselecter/querypreprocessor.cc @@ -1,7 +1,6 @@ #include "querypreprocessor.h" #include "core/index/index.h" -#include "core/index/indextext/indextext.h" #include "core/namespace/namespaceimpl.h" #include "core/nsselecter/joinedselector.h" #include "core/nsselecter/selectiteratorcontainer.h" @@ -9,7 +8,6 @@ #include "core/payload/fieldsset.h" #include "core/query/dsl/dslencoder.h" #include "core/query/queryentry.h" -#include "estl/overloaded.h" #include "nsselecter.h" #include "qresexplainholder.h" #include "substitutionhelpers.h" @@ -22,25 +20,23 @@ QueryPreprocessor::QueryPreprocessor(QueryEntries &&queries, NamespaceImpl *ns, query_{ctx.query}, strictMode_(ctx.inTransaction ? StrictModeNone : ((query_.strictMode == StrictModeNotSet) ? ns_.config_.strictMode : query_.strictMode)), - start_(query_.start), - count_(query_.count), + start_(query_.Offset()), + count_(query_.Limit()), forcedSortOrder_(!query_.forcedSortOrder_.empty()), reqMatchedOnce_(ctx.reqMatchedOnceFlag) { if (forcedSortOrder_ && (start_ > QueryEntry::kDefaultOffset || count_ < QueryEntry::kDefaultLimit)) { - assertrx(!query_.sortingEntries_.empty()); + assertrx_throw(!query_.sortingEntries_.empty()); static const std::vector emptyJoinedSelectors; const auto &sEntry = query_.sortingEntries_[0]; if (SortExpression::Parse(sEntry.expression, emptyJoinedSelectors).ByIndexField()) { - QueryEntry qe; - qe.values.reserve(query_.forcedSortOrder_.size()); - for (const auto &v : query_.forcedSortOrder_) qe.values.push_back(v); - qe.condition = query_.forcedSortOrder_.size() == 1 ? CondEq : CondSet; - qe.index = sEntry.expression; - if (!ns_.getIndexByNameOrJsonPath(qe.index, qe.idxNo)) { - qe.idxNo = IndexValueType::SetByJsonPath; - } + VariantArray values; + values.reserve(query_.forcedSortOrder_.size()); + for (const auto &v : query_.forcedSortOrder_) values.push_back(v); desc_ = sEntry.desc; - Append(desc_ ? OpNot : OpAnd, std::move(qe)); + QueryField fld{sEntry.expression}; + SetQueryField(fld, ns_); + Append(desc_ ? OpNot : OpAnd, std::move(fld), query_.forcedSortOrder_.size() == 1 ? CondEq : CondSet, + std::move(values)); queryEntryAddedByForcedSortOptimization_ = true; } } @@ -58,9 +54,8 @@ void QueryPreprocessor::ExcludeFtQuery(const RdxContext &rdxCtx) { if (queryEntryAddedByForcedSortOptimization_ || Size() <= 1) return; for (auto it = begin(), next = it, endIt = end(); it != endIt; it = next) { ++next; - if (it->HoldsOrReferTo() && it->Value().idxNo != IndexValueType::SetByJsonPath) { - const auto indexNo = it->Value().idxNo; - auto &index = ns_.indexes_[indexNo]; + if (it->HoldsOrReferTo() && it->Value().IsFieldIndexed()) { + auto &index = ns_.indexes_[it->Value().IndexNo()]; if (!IsFastFullText(index->Type())) continue; if (it->operation != OpAnd || (next != endIt && next->operation == OpOr) || !index->EnablePreselectBeforeFt()) break; ftPreselect_ = index->FtPreselect(rdxCtx); @@ -80,16 +75,16 @@ bool QueryPreprocessor::NeedNextEvaluation(unsigned start, unsigned count, bool if (evaluationsCount_++) return false; if (queryEntryAddedByForcedSortOptimization_) { container_.back().operation = desc_ ? OpAnd : OpNot; - assertrx(start <= start_); + assertrx_throw(start <= start_); start_ = start; - assertrx(count <= count_); + assertrx_throw(count <= count_); count_ = count; return count_ || (reqMatchedOnce_ && !matchedAtLeastOnce); } else if (ftEntry_) { if (!matchedAtLeastOnce) return false; qresHolder.BackupContainer(); - start_ = query_.start; - count_ = query_.count; + start_ = query_.Offset(); + count_ = query_.Limit(); forcedSortOrder_ = !query_.forcedSortOrder_.empty(); clear(); Append(OpAnd, std::move(*ftEntry_)); @@ -101,19 +96,19 @@ bool QueryPreprocessor::NeedNextEvaluation(unsigned start, unsigned count, bool return false; } -void QueryPreprocessor::checkStrictMode(const std::string &index, int idxNo) const { - if (idxNo != IndexValueType::SetByJsonPath) return; +void QueryPreprocessor::checkStrictMode(const QueryField &field) const { + if (field.IsFieldIndexed()) return; switch (strictMode_) { case StrictModeIndexes: - throw Error(errParams, + throw Error(errQueryExec, "Current query strict mode allows filtering by indexes only. There are no indexes with name '%s' in namespace '%s'", - index, ns_.name_); + field.FieldName(), ns_.name_); case StrictModeNames: - if (ns_.tagsMatcher_.path2tag(index).empty()) { - throw Error(errParams, + if (field.HaveEmptyField()) { + throw Error(errQueryExec, "Current query strict mode allows filtering by existing fields only. There are no fields with name '%s' in " "namespace '%s'", - index, ns_.name_); + field.FieldName(), ns_.name_); } case StrictModeNotSet: case StrictModeNone: @@ -151,7 +146,7 @@ bool QueryPreprocessor::removeBrackets() { bool QueryPreprocessor::canRemoveBracket(size_t i) const { if (Size(i) < 2) { - throw Error{errParams, "Bracket cannot be empty"}; + throw Error{errQueryExec, "Bracket cannot be empty"}; } const size_t next = Next(i); const OpType op = GetOperation(i); @@ -162,7 +157,7 @@ bool QueryPreprocessor::canRemoveBracket(size_t i) const { size_t QueryPreprocessor::removeBrackets(size_t begin, size_t end) { if (begin != end && GetOperation(begin) == OpOr) { - throw Error{errParams, "First condition cannot be with operation OR"}; + throw Error{errQueryExec, "OR operator in first condition or after left join"}; } size_t deleted = 0; for (size_t i = begin; i < end - deleted; i = Next(i)) { @@ -183,31 +178,23 @@ void QueryPreprocessor::InitIndexNumbers() { ExecuteAppropriateForEach( Skip{}, [this](QueryEntry &entry) { - if (entry.idxNo == IndexValueType::NotSet) { - if (!ns_.getIndexByNameOrJsonPath(entry.index, entry.idxNo)) { - entry.idxNo = IndexValueType::SetByJsonPath; - } + if (!entry.FieldsHaveBeenSet()) { + SetQueryField(entry.FieldData(), ns_); } - checkStrictMode(entry.index, entry.idxNo); + checkStrictMode(entry.FieldData()); }, [this](BetweenFieldsQueryEntry &entry) { - if (entry.firstIdxNo == IndexValueType::NotSet) { - if (!ns_.getIndexByNameOrJsonPath(entry.firstIndex, entry.firstIdxNo)) { - entry.firstIdxNo = IndexValueType::SetByJsonPath; - } - } - checkStrictMode(entry.firstIndex, entry.firstIdxNo); - if (entry.secondIdxNo == IndexValueType::NotSet) { - if (!ns_.getIndexByNameOrJsonPath(entry.secondIndex, entry.secondIdxNo)) { - entry.secondIdxNo = IndexValueType::SetByJsonPath; - } + if (!entry.FieldsHaveBeenSet()) { + SetQueryField(entry.LeftFieldData(), ns_); + SetQueryField(entry.RightFieldData(), ns_); } - checkStrictMode(entry.secondIndex, entry.secondIdxNo); + checkStrictMode(entry.LeftFieldData()); + checkStrictMode(entry.RightFieldData()); }); } size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, uint16_t srcEnd) { - assertrx(dst <= srcBegin); + assertrx_throw(dst <= srcBegin); h_vector iidx(kMaxIndexes, uint16_t(0)); size_t merged = 0; for (size_t src = srcBegin, nextSrc; src < srcEnd; src = nextSrc) { @@ -221,17 +208,16 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui return true; }, [&](QueryEntry &entry) { - const bool isIndexField = (entry.idxNo >= 0); - if (isIndexField) { + if (entry.IsFieldIndexed()) { // try merge entries with AND opetator if ((GetOperation(src) == OpAnd) && (nextSrc >= srcEnd || GetOperation(nextSrc) != OpOr)) { - if (size_t(entry.idxNo) >= iidx.size()) { + if (size_t(entry.IndexNo()) >= iidx.size()) { const auto oldSize = iidx.size(); - iidx.resize(size_t(entry.idxNo) + 1); + iidx.resize(entry.IndexNo() + 1); std::fill(iidx.begin() + oldSize, iidx.begin() + iidx.size(), 0); } - auto &iidxRef = iidx[entry.idxNo]; - if (iidxRef > 0 && !ns_.indexes_[entry.idxNo]->Opts().IsArray()) { + auto &iidxRef = iidx[entry.IndexNo()]; + if (iidxRef > 0 && !ns_.indexes_[entry.IndexNo()]->Opts().IsArray()) { if (mergeQueryEntries(iidxRef - 1, src)) { ++merged; return false; @@ -265,9 +251,9 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui void QueryPreprocessor::CheckUniqueFtQuery() const { bool found = false; ExecuteAppropriateForEach(Skip{}, [&](const QueryEntry &qe) { - if (qe.idxNo != IndexValueType::SetByJsonPath && IsFullText(ns_.indexes_[qe.idxNo]->Type())) { + if (qe.IsFieldIndexed() && IsFullText(ns_.indexes_[qe.IndexNo()]->Type())) { if (found) { - throw Error{errParams, "Query cannot contain more than one full text condition"}; + throw Error{errQueryExec, "Query cannot contain more than one full text condition"}; } else { found = true; } @@ -277,8 +263,8 @@ void QueryPreprocessor::CheckUniqueFtQuery() const { bool QueryPreprocessor::ContainsFullTextIndexes() const { for (auto it = cbegin().PlainIterator(), end = cend().PlainIterator(); it != end; ++it) { - if (it->HoldsOrReferTo() && it->Value().idxNo != IndexValueType::SetByJsonPath && - IsFullText(ns_.indexes_[it->Value().idxNo]->Type())) { + if (it->HoldsOrReferTo() && it->Value().IsFieldIndexed() && + IsFullText(ns_.indexes_[it->Value().IndexNo()]->Type())) { return true; } } @@ -321,11 +307,14 @@ static void createCompositeKeyValues(const h_vector } } -static void createCompositeKeyValues(const h_vector, 4> &values, const PayloadType &plType, - VariantArray &ret) { +static VariantArray createCompositeKeyValues(const h_vector, 4> &values, const PayloadType &plType, + uint32_t resultSetSize) { PayloadValue d(plType.TotalSize()); Payload pl(plType, d); + VariantArray ret; + ret.reserve(resultSetSize); createCompositeKeyValues(values, plType, pl, ret, 0); + return ret; } size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const size_t to) { @@ -350,15 +339,16 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si continue; } auto &qe = Get(cur); - if ((qe.condition != CondEq && qe.condition != CondSet) || qe.idxNo >= ns_.payloadType_.NumFields() || qe.idxNo < 0) { + if ((qe.Condition() != CondEq && qe.Condition() != CondSet) || !qe.IsFieldIndexed() || + qe.IndexNo() >= ns_.payloadType_.NumFields()) { continue; } - const std::vector *found = getCompositeIndex(qe.idxNo); + const std::vector *found = getCompositeIndex(qe.IndexNo()); if (!found || found->empty()) { continue; } - searcher.Add(qe.idxNo, *found, cur); + searcher.Add(qe.IndexNo(), *found, cur); } EntriesRanges deleteRanges; @@ -371,14 +361,13 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si uint32_t maxSetSize = 0; for (auto i : res.entries) { auto &qe = Get(i); - if rx_unlikely (!res.fields.contains(qe.idxNo)) { + if rx_unlikely (!res.fields.contains(qe.IndexNo())) { throw Error(errLogic, "Error during composite index's fields substitution (this should not happen)"); } - - maxSetSize = std::max(maxSetSize, qe.values.size()); - resultSetSize = (resultSetSize == 0) ? qe.values.size() : (resultSetSize * qe.values.size()); + maxSetSize = std::max(maxSetSize, qe.Values().size()); + resultSetSize = (resultSetSize == 0) ? qe.Values().size() : (resultSetSize * qe.Values().size()); } - static const CompositeValuesCountLimits kCompositeSetLimits; + constexpr static CompositeValuesCountLimits kCompositeSetLimits; if (resultSetSize != maxSetSize) { // Do not perform substitution if result set size becoms larger than initial indexes set size // and this size is greater than limit @@ -390,22 +379,17 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si } for (auto i : res.entries) { auto &qe = Get(i); - const auto idxKeyType = ns_.indexes_[qe.idxNo]->KeyType(); - for (auto &v : qe.values) { - v.convert(idxKeyType); - } - values.emplace_back(qe.idxNo, std::move(qe.values)); + qe.ConvertValuesToFieldType(); + const int idxNo = qe.IndexNo(); + values.emplace_back(idxNo, std::move(qe).Values()); } { - QueryEntry ce(CondSet, ns_.indexes_[res.idx]->Name(), res.idx); - ce.values.reserve(resultSetSize); - createCompositeKeyValues(values, ns_.payloadType_, ce.values); - if (ce.values.size() == 1) { - ce.condition = CondEq; - } + VariantArray qValues = createCompositeKeyValues(values, ns_.payloadType_, resultSetSize); const auto first = res.entries.front(); SetOperation(OpAnd, first); - container_[first].SetValue(std::move(ce)); + QueryField fld{ns_.indexes_[res.idx]->Name()}; + setQueryIndex(fld, res.idx, ns_); + container_[first].Emplace(std::move(fld), qValues.size() == 1 ? CondEq : CondSet, std::move(qValues)); } deleteRanges.Add(span(res.entries.data() + 1, res.entries.size() - 1)); resIdx = searcher.RemoveUsedAndGetNext(resIdx); @@ -417,29 +401,14 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si return deleted; } -void QueryPreprocessor::convertWhereValues(QueryEntry *qe) const { - const FieldsSet *fields = nullptr; - KeyValueType keyType{KeyValueType::Undefined{}}; - const bool isIndexField = (qe->idxNo != IndexValueType::SetByJsonPath); - if (isIndexField) { - keyType = ns_.indexes_[qe->idxNo]->SelectKeyType(); - fields = &ns_.indexes_[qe->idxNo]->Fields(); - } - if (!keyType.Is()) { - if (qe->condition != CondDWithin) { - for (auto &key : qe->values) { - key.convert(keyType, &ns_.payloadType_, fields); - } - } - } -} +void QueryPreprocessor::convertWhereValues(QueryEntry &qe) const { qe.ConvertValuesToFieldType(ns_.payloadType_); } void QueryPreprocessor::convertWhereValues(QueryEntries::iterator begin, QueryEntries::iterator end) const { for (auto it = begin; it != end; ++it) { it->InvokeAppropriate( Skip{}, [this, &it](const QueryEntriesBracket &) { convertWhereValues(it.begin(), it.end()); }, - [this](QueryEntry &qe) { convertWhereValues(&qe); }); + [this](QueryEntry &qe) { convertWhereValues(qe); }); } } @@ -481,12 +450,12 @@ void QueryPreprocessor::findMaxIndex(QueryEntries::const_iterator begin, QueryEn return FoundIndexInfo(); }, [this](const QueryEntry &entry) -> FoundIndexInfo { - if (entry.idxNo != IndexValueType::SetByJsonPath && !entry.distinct) { - const auto idxPtr = ns_.indexes_[entry.idxNo].get(); + if (entry.IsFieldIndexed() && !entry.Distinct()) { + const auto idxPtr = ns_.indexes_[entry.IndexNo()].get(); if (idxPtr->IsOrdered() && !idxPtr->Opts().IsArray()) { - if (IsOrderedCondition(entry.condition)) { + if (IsOrderedCondition(entry.Condition())) { return FoundIndexInfo{idxPtr, FoundIndexInfo::ConditionType::Compatible}; - } else if (entry.condition == CondAny || entry.values.size() > 1) { + } else if (entry.Condition() == CondAny || entry.Values().size() > 1) { return FoundIndexInfo{idxPtr, FoundIndexInfo::ConditionType::Incompatible}; } } @@ -510,42 +479,42 @@ void QueryPreprocessor::findMaxIndex(QueryEntries::const_iterator begin, QueryEn bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { QueryEntry *lqe = &Get(lhs); QueryEntry &rqe = Get(rhs); - if ((lqe->condition == CondEq || lqe->condition == CondSet) && (rqe.condition == CondEq || rqe.condition == CondSet)) { + if ((lqe->Condition() == CondEq || lqe->Condition() == CondSet) && (rqe.Condition() == CondEq || rqe.Condition() == CondSet)) { // intersect 2 queryentries on the same index - if rx_unlikely (lqe->values.empty()) { + if rx_unlikely (lqe->Values().empty()) { return true; } if (container_[lhs].IsRef()) { container_[lhs].SetValue(const_cast(*lqe)); lqe = &Get(lhs); } + const bool distinct = lqe->Distinct() || rqe.Distinct(); VariantArray setValues; - if (rx_likely(!rqe.values.empty())) { - convertWhereValues(lqe); - convertWhereValues(&rqe); - VariantArray *first = &lqe->values; - VariantArray *second = &rqe.values; - if (lqe->values.size() > rqe.values.size()) { - std::swap(first, second); - } - setValues.reserve(first->size()); + if (rx_likely(!rqe.Values().empty())) { + convertWhereValues(*lqe); + convertWhereValues(rqe); + auto &&[first, second] = lqe->Values().size() < rqe.Values().size() + ? std::make_pair(std::move(*lqe).Values(), std::move(rqe).Values()) + : std::make_pair(std::move(rqe).Values(), std::move(*lqe).Values()); + + setValues.reserve(first.size()); constexpr size_t kMinArraySizeToUseHashSet = 250; - if (second->size() < kMinArraySizeToUseHashSet) { + if (second.size() < kMinArraySizeToUseHashSet) { // Intersect via binary search + sort for small vectors - std::sort(first->begin(), first->end()); - for (auto &&v : *second) { - if (std::binary_search(first->begin(), first->end(), v)) { + std::sort(first.begin(), first.end()); + for (auto &&v : second) { + if (std::binary_search(first.begin(), first.end(), v)) { setValues.emplace_back(std::move(v)); } } } else { // Intersect via hash_set for large vectors reindexer::fast_hash_set set; - set.reserve(first->size() * 2); - for (auto &&v : *first) { + set.reserve(first.size() * 2); + for (auto &&v : first) { set.emplace(std::move(v)); } - for (auto &&v : *second) { + for (auto &&v : second) { if (set.erase(v)) { setValues.emplace_back(std::move(v)); } @@ -553,27 +522,26 @@ bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { } } - lqe->values = std::move(setValues); - lqe->condition = (lqe->values.size() == 1) ? CondEq : CondSet; - lqe->distinct |= rqe.distinct; + lqe->SetCondAndValues(CondSet, std::move(setValues)); + lqe->Distinct(distinct); return true; - } else if (rqe.condition == CondAny) { - if (!lqe->distinct && rqe.distinct) { + } else if (rqe.Condition() == CondAny) { + if (!lqe->Distinct() && rqe.Distinct()) { if (container_[lhs].IsRef()) { container_[lhs].SetValue(const_cast(*lqe)); lqe = &Get(lhs); } - lqe->distinct = true; + lqe->Distinct(true); } return true; - } else if (lqe->condition == CondAny) { - const bool distinct = lqe->distinct || rqe.distinct; + } else if (lqe->Condition() == CondAny) { + const bool distinct = lqe->Distinct() || rqe.Distinct(); if (container_[rhs].IsRef()) { container_[lhs].SetValue(const_cast(rqe)); } else { container_[lhs].SetValue(std::move(rqe)); } - Get(lhs).distinct = distinct; + Get(lhs).Distinct(distinct); return true; } @@ -584,19 +552,16 @@ void QueryPreprocessor::AddDistinctEntries(const h_vector &aggreg bool wasAdded = false; for (auto &ag : aggregators) { if (ag.Type() != AggDistinct) continue; - QueryEntry qe; - assertrx(ag.Names().size() == 1); - qe.index = ag.Names()[0]; - qe.condition = CondAny; - qe.distinct = true; - Append(wasAdded ? OpOr : OpAnd, std::move(qe)); + assertrx_throw(ag.Names().size() == 1); + Append(wasAdded ? OpOr : OpAnd, ag.Names()[0], QueryEntry::DistinctTag{}); wasAdded = true; } } -void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, std::string &explainStr, AggType &oAggType, - NamespaceImpl &rightNs, Query joinQuery, std::string joinIndex, CondType condition, - KeyValueType valuesType, const RdxContext &rdxCtx) { +std::pair QueryPreprocessor::queryValuesFromOnCondition(std::string &explainStr, AggType &oAggType, + NamespaceImpl &rightNs, Query joinQuery, + const QueryJoinEntry &joinEntry, CondType condition, + const RdxContext &rdxCtx) { size_t limit; const auto &rNsCfg = rightNs.Config(); if (rNsCfg.maxPreselectSize == 0) { @@ -608,25 +573,25 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, st std::min(std::max(rNsCfg.minPreselectSize, rightNs.ItemsCount() * rNsCfg.maxPreselectPart), rNsCfg.maxPreselectSize); } joinQuery.explain_ = query_.explain_; - joinQuery.count = limit + 2; - joinQuery.start = 0; + joinQuery.Limit(limit + 2); + joinQuery.Offset(QueryEntry::kDefaultOffset); joinQuery.sortingEntries_.clear(); joinQuery.forcedSortOrder_.clear(); joinQuery.aggregations_.clear(); switch (condition) { case CondEq: case CondSet: - joinQuery.Distinct(std::move(joinIndex)); + joinQuery.Distinct(joinEntry.RightFieldName()); oAggType = AggType::AggDistinct; break; case CondLt: case CondLe: - joinQuery.Aggregate(AggMax, {std::move(joinIndex)}); + joinQuery.Aggregate(AggMax, {joinEntry.RightFieldName()}); oAggType = AggType::AggMax; break; case CondGt: case CondGe: - joinQuery.Aggregate(AggMin, {std::move(joinIndex)}); + joinQuery.Aggregate(AggMin, {joinEntry.RightFieldName()}); oAggType = AggType::AggMin; break; case CondAny: @@ -635,104 +600,87 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, st case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } SelectCtx ctx{joinQuery, nullptr}; QueryResults qr; rightNs.Select(qr, ctx, rdxCtx); - if (qr.Count() > limit) return; - assertrx(qr.aggregationResults.size() == 1); + if (qr.Count() > limit) return {CondAny, {}}; + assertrx_throw(qr.aggregationResults.size() == 1); + auto &aggRes = qr.aggregationResults[0]; explainStr = qr.explainResults; switch (condition) { case CondEq: case CondSet: { - assertrx(qr.aggregationResults[0].type == AggDistinct); - queryEntry.values.reserve(qr.aggregationResults[0].distincts.size()); - assertrx(qr.aggregationResults[0].distinctsFields.size() == 1); - const auto field = qr.aggregationResults[0].distinctsFields[0]; - for (Variant &distValue : qr.aggregationResults[0].distincts) { + assertrx_throw(aggRes.type == AggDistinct); + VariantArray values; + values.reserve(aggRes.distincts.size()); + for (Variant &distValue : aggRes.distincts) { if (distValue.Type().Is()) { - ConstPayload pl(qr.aggregationResults[0].payloadType, distValue.operator const PayloadValue &()); - VariantArray v; - if (field == IndexValueType::SetByJsonPath) { - assertrx(qr.aggregationResults[0].distinctsFields.getTagsPathsLength() == 1); - pl.GetByJsonPath(qr.aggregationResults[0].distinctsFields.getTagsPath(0), v, valuesType); - } else { - pl.Get(field, v); - } - assertrx(v.size() == 1); - queryEntry.values.emplace_back(std::move(v[0])); + ConstPayload pl(aggRes.payloadType, distValue.operator const PayloadValue &()); + values.emplace_back(pl.GetComposite(aggRes.distinctsFields, joinEntry.RightCompositeFieldsTypes())); } else { - queryEntry.values.emplace_back(std::move(distValue)); + values.emplace_back(std::move(distValue)); } } - queryEntry.condition = (queryEntry.values.size() == 1) ? CondEq : CondSet; - break; + return {CondSet, std::move(values)}; } case CondLt: case CondLe: case CondGt: case CondGe: - if (auto value = qr.aggregationResults[0].GetValue()) { - queryEntry.condition = condition; - queryEntry.values.emplace_back(*value); + if (auto value = aggRes.GetValue()) { + return {condition, {Variant{*value}}}; + } else { + return {CondAny, {}}; } - break; case CondAny: case CondRange: case CondAllSet: case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + default: + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } } -template -void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, std::string_view joinIndex, CondType condition, - const JoinedSelector &joinedSelector, KeyValueType valuesType, const int rightIdxNo, - const CollateOpts &collate) { - JoinPreResult::Values &values = joinedSelector.preResult_->values; +std::pair QueryPreprocessor::queryValuesFromOnCondition(CondType condition, const QueryJoinEntry &joinEntry, + const JoinedSelector &joinedSelector, + const CollateOpts &collate) { switch (condition) { case CondEq: - case CondSet: { - joinedSelector.readValuesFromPreResult(queryEntry.values, valuesType, rightIdxNo, joinIndex); - queryEntry.condition = (queryEntry.values.size() == 1) ? CondEq : CondSet; - return; - } + case CondSet: + return {CondSet, joinedSelector.readValuesFromPreResult(joinEntry)}; case CondLt: case CondLe: case CondGt: case CondGe: { - queryEntry.condition = condition; - VariantArray buffer; + const JoinPreResult::Values &values = joinedSelector.preResult_->values; + VariantArray buffer, keyValues; for (const ItemRef &item : values) { - buffer.clear(); - assertrx(!item.Value().IsFree()); + assertrx_throw(!item.Value().IsFree()); const ConstPayload pl{values.payloadType, item.Value()}; - if constexpr (byJsonPath) { - pl.GetByJsonPath(joinIndex, values.tagsMatcher, buffer, valuesType); - } else { - pl.Get(rightIdxNo, buffer); - } + pl.GetByFieldsSet(joinEntry.RightFields(), buffer, joinEntry.RightFieldType(), joinEntry.RightCompositeFieldsTypes()); for (Variant &v : buffer) { - if (queryEntry.values.empty()) { - queryEntry.values.emplace_back(std::move(v)); + if (keyValues.empty()) { + keyValues.emplace_back(std::move(v)); } else { - const auto cmp = queryEntry.values[0].Compare(v, collate); + const auto cmp = keyValues[0].Compare(v, collate); if (condition == CondLt || condition == CondLe) { if (cmp < 0) { - queryEntry.values[0] = std::move(v); + keyValues[0] = std::move(v); } } else { if (cmp > 0) { - queryEntry.values[0] = std::move(v); + keyValues[0] = std::move(v); } } } } } + return {condition, std::move(keyValues)}; } break; case CondAny: case CondRange: @@ -740,7 +688,8 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, st case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + default: + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } } @@ -782,8 +731,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join assertrx_throw(to <= container_.size()); }, [&](const JoinQueryEntry &jqe) { - assertrx(js.size() > jqe.joinIndex); - + assertrx_throw(js.size() > jqe.joinIndex); JoinedSelector &joinedSelector = js[jqe.joinIndex]; const bool byValues = joinedSelector.PreResult() && joinedSelector.PreResult()->dataMode == JoinPreResult::ModeValues; @@ -803,14 +751,13 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join return; } } - const auto &joinEntries = joinedSelector.joinQuery_.joinEntries_; // LeftJoin-s shall not be in QueryEntries container_ by construction - assertrx(joinedSelector.Type() == InnerJoin || joinedSelector.Type() == OrInnerJoin); + assertrx_throw(joinedSelector.Type() == InnerJoin || joinedSelector.Type() == OrInnerJoin); // Checking if we have anything to inject into main Where clause bool foundANDOrOR = false; for (const auto &je : joinEntries) { - if (je.op_ != OpNot) { + if (je.Operation() != OpNot) { foundANDOrOR = true; break; } @@ -822,7 +769,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join OpType op = GetOperation(cur); if (joinedSelector.Type() == OrInnerJoin) { - if (op == OpNot) throw Error(errParams, "OR INNER JOIN with operation NOT"); + if (op == OpNot) throw Error(errQueryExec, "OR INNER JOIN with operation NOT"); op = OpOr; joinedSelector.SetType(InnerJoin); } @@ -832,6 +779,8 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join // !!!Warning jqe reference will be invalidated after EncloseInBracket EncloseInBracket(cur, cur + 1, op); ++cur; + ++to; + ++injectedCount; explainJoinOn.ReserveOnEntries(joinEntries.size()); @@ -840,12 +789,10 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join size_t orChainLength = 0; for (size_t i = 0, s = joinEntries.size(); i < s; ++i) { const QueryJoinEntry &joinEntry = joinEntries[i]; - auto explainEntry = explainJoinOn.AppendOnEntryExplain(); explainEntry.InitialCondition(joinEntry, joinedSelector); - - CondType condition = joinEntry.condition_; - OpType operation = joinEntry.op_; + CondType condition = joinEntry.Condition(); + OpType operation = joinEntry.Operation(); switch (operation) { case OpNot: orChainLength = 0; @@ -873,7 +820,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } operation = OpAnd; break; @@ -888,67 +835,67 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join orChainLength = 0; break; } - - QueryEntry newEntry; - newEntry.index = joinEntry.index_; - newEntry.idxNo = IndexValueType::SetByJsonPath; - KeyValueType valuesType = KeyValueType::Undefined{}; - CollateOpts collate; - if (ns_.getIndexByNameOrJsonPath(newEntry.index, newEntry.idxNo)) { - const Index &index = *ns_.indexes_[newEntry.idxNo]; - valuesType = index.SelectKeyType(); - collate = index.Opts().collateOpts_; - } - + CondType queryCondition{CondAny}; + VariantArray values; if (byValues) { - assertrx(joinedSelector.itemQuery_.entries.HoldsOrReferTo(i)); - const QueryEntry &qe = joinedSelector.itemQuery_.entries.Get(i); - assertrx(qe.index == joinEntry.joinIndex_); - const int rightIdxNo = qe.idxNo; - if (rightIdxNo == IndexValueType::SetByJsonPath) { - fillQueryEntryFromOnCondition(newEntry, joinEntry.joinIndex_, condition, joinedSelector, valuesType, - rightIdxNo, collate); - } else { - fillQueryEntryFromOnCondition(newEntry, joinEntry.joinIndex_, condition, joinedSelector, valuesType, - rightIdxNo, collate); + assertrx_throw(joinedSelector.itemQuery_.entries.HoldsOrReferTo(i)); + assertrx_throw(joinedSelector.itemQuery_.entries.Get(i).FieldName() == joinEntry.RightFieldName()); + CollateOpts collate; + if (joinEntry.IsLeftFieldIndexed()) { + collate = ns_.indexes_[joinEntry.LeftIdxNo()]->Opts().collateOpts_; } + std::tie(queryCondition, values) = queryValuesFromOnCondition(condition, joinEntry, joinedSelector, collate); } else { bool skip = false; switch (condition) { + case CondAny: + case CondEmpty: + case CondLike: + case CondDWithin: + explainEntry.Skipped("Skipped due to unsupperted on condition"sv); + skip = true; + break; + case CondRange: case CondLt: case CondLe: case CondGt: - case CondGe: { - const QueryEntry &qe = joinedSelector.itemQuery_.entries.Get(i); - skip = qe.idxNo != IndexValueType::SetByJsonPath && joinedSelector.RightNs()->indexes_[qe.idxNo]->IsUuid(); - if (skip) { - explainEntry.Skipped("Skipped due to condition Lt|Le|Gt|Ge with UUID index field."sv); - } + case CondGe: + joinedSelector.itemQuery_.entries.Get(i).FieldType().EvaluateOneOf( + [&skip, + &explainEntry](OneOf) noexcept { + skip = true; + explainEntry.Skipped( + "Skipped due to condition Lt|Le|Gt|Ge|Range with not indexed or not numeric field."sv); + }, + [](OneOf) noexcept { + }); break; - } case CondEq: case CondSet: case CondAllSet: - case CondAny: - case CondEmpty: - case CondRange: - case CondLike: - case CondDWithin: + joinedSelector.itemQuery_.entries.Get(i).FieldType().EvaluateOneOf( + [&skip, &explainEntry](OneOf) noexcept { + skip = true; + explainEntry.Skipped("Skipped due to condition Eq|Set|AllSet with composite index."sv); + }, + [](OneOf) noexcept {}); break; } if (!skip) { std::string explainSelect; AggType selectAggType; - fillQueryEntryFromOnCondition(newEntry, explainSelect, selectAggType, *joinedSelector.RightNs(), - joinedSelector.JoinQuery(), joinEntry.joinIndex_, condition, valuesType, rdxCtx); + std::tie(queryCondition, values) = + queryValuesFromOnCondition(explainSelect, selectAggType, *joinedSelector.RightNs(), + joinedSelector.JoinQuery(), joinEntry, condition, rdxCtx); explainEntry.ExplainSelect(std::move(explainSelect), selectAggType); } } - - if (!newEntry.values.empty()) { - explainEntry.Succeed(newEntry); - - Insert(cur, operation, std::move(newEntry)); + if (!values.empty()) { + Insert(cur, operation, QueryEntry{QueryField(joinEntry.LeftFieldData()), queryCondition, std::move(values)}); + explainEntry.Succeed(Get(cur)); ++cur; ++count; prevIsSkipped = false; @@ -956,6 +903,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join explainEntry.Skipped("Skipped as cannot obtain values from right namespace."sv); if (operation == OpOr) { Erase(cur - orChainLength, cur); + cur -= orChainLength; count -= orChainLength; // Marking On-injections as fail for removed entries. explainJoinOn.FailOnEntriesAsOrChain(orChainLength); @@ -971,8 +919,10 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join [this, cur, count, &js](WrSerializer &ser) { briefDump(cur - count, Next(cur - count), js, ser); }); ++cur; - injectedCount += count + 2; - to += count + 2; + injectedCount += count + 1; + to += count + 1; + } else { + explainJoinOn.Skipped("Skipped as there are no injected conditions"); } }); } @@ -1020,7 +970,7 @@ class JoinOnExplainEnabled { explainEntry_.succeed = true; explainEntry_.reason = ""; explainEntry_.newCond = newEntry.DumpBrief(); - explainEntry_.valuesCount = newEntry.values.size(); + explainEntry_.valuesCount = newEntry.Values().size(); } void Skipped(std::string_view reason) noexcept { @@ -1090,4 +1040,34 @@ class JoinOnExplainEnabled { time_point_t startTime_; }; +void QueryPreprocessor::setQueryIndex(QueryField &qField, int idxNo, const NamespaceImpl &ns) { + const auto &idx = *ns.indexes_[idxNo]; + std::vector compositeFieldsTypes; + if (idxNo >= ns.indexes_.firstCompositePos()) { +#ifndef NDEBUG + const bool ftIdx = IsFullText(idx.Type()); +#endif + for (const auto f : ns.indexes_[idxNo]->Fields()) { + if (f == IndexValueType::SetByJsonPath) { + // not indexed fields allowed only in ft composite indexes + assertrx_throw(ftIdx); + compositeFieldsTypes.push_back(KeyValueType::String{}); + } else { + assertrx_throw(f <= ns.indexes_.firstCompositePos()); + compositeFieldsTypes.push_back(ns.indexes_[f]->SelectKeyType()); + } + } + } + qField.SetIndexData(idxNo, FieldsSet(idx.Fields()), idx.KeyType(), idx.SelectKeyType(), std::move(compositeFieldsTypes)); +} + +void QueryPreprocessor::SetQueryField(QueryField &qField, const NamespaceImpl &ns) { + int idxNo = IndexValueType::SetByJsonPath; + if (ns.getIndexByNameOrJsonPath(qField.FieldName(), idxNo)) { + setQueryIndex(qField, idxNo, ns); + } else { + qField.SetField({ns.tagsMatcher_.path2tag(qField.FieldName())}); + } +} + } // namespace reindexer diff --git a/cpp_src/core/nsselecter/querypreprocessor.h b/cpp_src/core/nsselecter/querypreprocessor.h index 907636251..ba79d4c10 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.h +++ b/cpp_src/core/nsselecter/querypreprocessor.h @@ -65,6 +65,7 @@ class QueryPreprocessor : private QueryEntries { return std::move(*ftPreselect_); } bool IsFtPreselected() const noexcept { return ftPreselect_ && !ftEntry_; } + static void SetQueryField(QueryField &, const NamespaceImpl &); private: struct FoundIndexInfo { @@ -78,6 +79,7 @@ class QueryPreprocessor : private QueryEntries { uint64_t isFitForSortOptimization : 1; }; + static void setQueryIndex(QueryField &, int idxNo, const NamespaceImpl &); [[nodiscard]] SortingEntries detectOptimalSortOrder() const; bool forcedStage() const noexcept { return evaluationsCount_ == (desc_ ? 1 : 0); } size_t lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, uint16_t srcEnd); @@ -85,7 +87,7 @@ class QueryPreprocessor : private QueryEntries { bool mergeQueryEntries(size_t lhs, size_t rhs); const std::vector *getCompositeIndex(int field) const; void convertWhereValues(QueryEntries::iterator begin, QueryEntries::iterator end) const; - void convertWhereValues(QueryEntry *) const; + void convertWhereValues(QueryEntry &) const; [[nodiscard]] const Index *findMaxIndex(QueryEntries::const_iterator begin, QueryEntries::const_iterator end) const; void findMaxIndex(QueryEntries::const_iterator begin, QueryEntries::const_iterator end, h_vector &foundIndexes) const; @@ -94,12 +96,13 @@ class QueryPreprocessor : private QueryEntries { */ template size_t injectConditionsFromJoins(size_t from, size_t to, JoinedSelectors &, OnConditionInjections &, const RdxContext &); - void fillQueryEntryFromOnCondition(QueryEntry &, std::string &outExplainStr, AggType &, NamespaceImpl &rightNs, Query joinQuery, - std::string joinIndex, CondType condition, KeyValueType, const RdxContext &); - template - void fillQueryEntryFromOnCondition(QueryEntry &, std::string_view joinIndex, CondType condition, const JoinedSelector &, KeyValueType, - int rightIdxNo, const CollateOpts &); - void checkStrictMode(const std::string &index, int idxNo) const; + [[nodiscard]] std::pair queryValuesFromOnCondition(std::string &outExplainStr, AggType &, + NamespaceImpl &rightNs, Query joinQuery, + const QueryJoinEntry &, CondType condition, + const RdxContext &); + [[nodiscard]] std::pair queryValuesFromOnCondition(CondType condition, const QueryJoinEntry &, + const JoinedSelector &, const CollateOpts &); + void checkStrictMode(const QueryField &) const; bool removeBrackets(); size_t removeBrackets(size_t begin, size_t end); bool canRemoveBracket(size_t i) const; diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.cc b/cpp_src/core/nsselecter/selectiteratorcontainer.cc index a926d6fb9..f34f73137 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.cc +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.cc @@ -188,46 +188,15 @@ void SelectIteratorContainer::SetExpectMaxIterations(int expectedIterations) { SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe, const NamespaceImpl &ns, StrictMode strictMode) { SelectKeyResults selectResults; - FieldsSet fields; - TagsPath tagsPath = ns.tagsMatcher_.path2tag(qe.index); - - // TODO: it may be necessary to remove or change this switch after QueryEntry refactoring - switch (qe.condition) { - case CondAny: - case CondEmpty: - case CondAllSet: - case CondEq: - case CondSet: - break; - case CondRange: - case CondDWithin: - if (qe.values.size() != 2) { - throw Error(errParams, "For condition %s required exactly 2 arguments, but provided %d", CondTypeToStr(qe.condition), - qe.values.size()); - } - break; - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - if (qe.values.size() != 1) { - throw Error(errParams, "For condition %s required exactly 1 argument, but provided %d", CondTypeToStr(qe.condition), - qe.values.size()); - } - break; - } - - if (!tagsPath.empty()) { + if (!qe.HaveEmptyField()) { SelectKeyResult comparisonResult; - fields.push_back(tagsPath); - comparisonResult.comparators_.emplace_back(qe.condition, KeyValueType::Null{}, qe.values, false, qe.distinct, ns.payloadType_, - fields, nullptr, CollateOpts()); + comparisonResult.comparators_.emplace_back(qe.Condition(), KeyValueType::Null{}, qe.Values(), false, qe.Distinct(), ns.payloadType_, + qe.Fields(), nullptr, CollateOpts()); selectResults.emplace_back(std::move(comparisonResult)); } else if (strictMode == StrictModeNone) { SelectKeyResult res; // Ignore non-index/non-existing fields - if (qe.condition == CondEmpty) { + if (qe.Condition() == CondEmpty) { res.emplace_back(SingleSelectKeyResult(IdType(0), IdType(ns.items_.size()))); } else { res.emplace_back(SingleSelectKeyResult(IdType(0), IdType(0))); @@ -237,33 +206,30 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe throw Error( errParams, "Current query strict mode allows filtering by existing fields only. There are no fields with name '%s' in namespace '%s'", - qe.index, ns.name_); + qe.FieldName(), ns.name_); } return selectResults; } template -void SelectIteratorContainer::processField(FieldsComparator &fc, std::string_view field, int idxNo, const NamespaceImpl &ns) const { - const bool nonIndexField = (idxNo == IndexValueType::SetByJsonPath); - if (nonIndexField) { - TagsPath tagsPath = ns.tagsMatcher_.path2tag(field); - if (tagsPath.empty()) { - throw Error{errQueryExec, "Only existing fields can be compared. There are no fields with name '%s' in namespace '%s'", field, - ns.name_}; - } +void SelectIteratorContainer::processField(FieldsComparator &fc, const QueryField &field, const NamespaceImpl &ns) const { + if (field.IsFieldIndexed()) { + auto &index = ns.indexes_[field.IndexNo()]; if constexpr (left) { - fc.SetLeftField(tagsPath); + fc.SetCollateOpts(index->Opts().collateOpts_); + fc.SetLeftField(field.Fields(), field.FieldType(), index->Opts().IsArray()); } else { - fc.SetRightField(tagsPath); + fc.SetRightField(field.Fields(), field.FieldType(), index->Opts().IsArray()); } + } else if (field.HaveEmptyField()) { + throw Error{errQueryExec, "Only existing fields can be compared. There are no fields with name '%s' in namespace '%s'", + field.FieldName(), ns.name_}; } else { - auto &index = ns.indexes_[idxNo]; if constexpr (left) { - fc.SetCollateOpts(index->Opts().collateOpts_); - fc.SetLeftField(index->Fields(), index->KeyType(), index->Opts().IsArray()); + fc.SetLeftField(field.Fields()); } else { - fc.SetRightField(index->Fields(), index->KeyType(), index->Opts().IsArray()); + fc.SetRightField(field.Fields()); } } } @@ -272,7 +238,7 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe unsigned sortId, bool isQueryFt, SelectFunction::Ptr &selectFnc, bool &isIndexFt, bool &isIndexSparse, FtCtx::Ptr &ftCtx, QueryPreprocessor &qPreproc, const RdxContext &rdxCtx) { - auto &index = ns.indexes_[qe.idxNo]; + auto &index = ns.indexes_[qe.IndexNo()]; isIndexFt = IsFullText(index->Type()); isIndexSparse = index->Opts().IsSparse(); @@ -289,31 +255,31 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe opts.forceComparator = 1; } } - if (qe.distinct) { + if (qe.Distinct()) { opts.distinct = 1; } opts.maxIterations = GetMaxIterations(); opts.indexesNotOptimized = !ctx_->sortingContext.enableSortOrders; opts.inTransaction = ctx_->inTransaction; - auto ctx = selectFnc ? selectFnc->CreateCtx(qe.idxNo) : BaseFunctionCtx::Ptr{}; + auto ctx = selectFnc ? selectFnc->CreateCtx(qe.IndexNo()) : BaseFunctionCtx::Ptr{}; if (ctx && ctx->type == BaseFunctionCtx::kFtCtx) ftCtx = reindexer::reinterpret_pointer_cast(ctx); if (index->Opts().GetCollateMode() == CollateUTF8 || isIndexFt) { - for (auto &key : qe.values) key.EnsureUTF8(); + for (auto &key : qe.Values()) key.EnsureUTF8(); } PerfStatCalculatorMT calc(index->GetSelectPerfCounter(), ns.enablePerfCounters_); if (qPreproc.IsFtPreselected()) { - return index->SelectKey(qe.values, qe.condition, opts, ctx, qPreproc.MoveFtPreselect(), rdxCtx); + return index->SelectKey(qe.Values(), qe.Condition(), opts, ctx, qPreproc.MoveFtPreselect(), rdxCtx); } else { - return index->SelectKey(qe.values, qe.condition, sortId, opts, ctx, rdxCtx); + return index->SelectKey(qe.Values(), qe.Condition(), sortId, opts, ctx, rdxCtx); } } void SelectIteratorContainer::processJoinEntry(const JoinQueryEntry &jqe, OpType op) { auto &js = (*ctx_->joinedSelectors)[jqe.joinIndex]; if (js.JoinQuery().joinEntries_.empty()) throw Error(errQueryExec, "Join without ON conditions"); - if (js.JoinQuery().joinEntries_[0].op_ == OpOr) throw Error(errQueryExec, "The first ON condition cannot have OR operation"); + if (js.JoinQuery().joinEntries_[0].Operation() == OpOr) throw Error(errQueryExec, "The first ON condition cannot have OR operation"); if (js.Type() != InnerJoin && js.Type() != OrInnerJoin) throw Error(errLogic, "Not INNER JOIN in QueryEntry"); if (js.Type() == OrInnerJoin) { if (op == OpNot) throw Error(errQueryExec, "NOT operator with or_inner_join"); @@ -327,7 +293,7 @@ void SelectIteratorContainer::processJoinEntry(const JoinQueryEntry &jqe, OpType } void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectResults, OpType op, const NamespaceImpl &ns, - const QueryEntry &qe, bool isIndexFt, bool isIndexSparse, bool nonIndexField, + const QueryEntry &qe, bool isIndexFt, bool isIndexSparse, std::optional nextOp) { if (selectResults.empty()) { if (op == OpAnd) { @@ -340,19 +306,19 @@ void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectR case OpOr: { const iterator last = lastAppendedOrClosed(); if (last == this->end()) { - throw Error(errQueryExec, "OR operator in first condition or after left join "); + throw Error(errQueryExec, "OR operator in first condition or after left join"); } if (last->HoldsOrReferTo() && !last->Value().distinct && last->operation != OpNot) { if (last->IsRef()) { last->SetValue(last->Value()); } SelectIterator &it = last->Value(); - if (nonIndexField || isIndexSparse) { + if (!qe.IsFieldIndexed() || isIndexSparse) { it.Append(res); } else { - it.AppendAndBind(res, ns.payloadType_, qe.idxNo); + it.AppendAndBind(res, ns.payloadType_, qe.IndexNo()); } - it.name += " or " + qe.index; + it.name += " or " + qe.FieldName(); break; } } @@ -360,16 +326,16 @@ void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectR case OpNot: case OpAnd: // Iterator Field Kind: Query entry results. Field known. - Append(op, SelectIterator(res, qe.distinct, qe.index, - qe.idxNo < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed, isIndexFt)); - if (!nonIndexField && !isIndexSparse) { + Append(op, res, qe.Distinct(), qe.FieldName(), + qe.IndexNo() < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed, isIndexFt); + if (qe.IsFieldIndexed() && !isIndexSparse) { // last appended is always a SelectIterator const auto lastAppendedIt = lastAppendedOrClosed(); if (lastAppendedIt->IsRef()) { lastAppendedIt->SetValue(lastAppendedIt->Value()); } SelectIterator &lastAppended = lastAppendedIt->Value(); - lastAppended.Bind(ns.payloadType_, qe.idxNo); + lastAppended.Bind(ns.payloadType_, qe.IndexNo()); lastAppended.SetNotOperationFlag(op == OpNot); const auto maxIterations = lastAppended.GetMaxIterations(); const int cur = op == OpNot ? ns.items_.size() - maxIterations : maxIterations; @@ -393,25 +359,23 @@ void SelectIteratorContainer::processEqualPositions(const std::vector(eqPos.queryEntriesPositions[0])}; - if (firstQe.condition == CondEmpty || (firstQe.condition == CondSet && firstQe.values.empty())) { + if (firstQe.Condition() == CondEmpty || (firstQe.Condition() == CondSet && firstQe.Values().empty())) { throw Error(errLogic, "Condition IN(with empty parameter list), IS NULL, IS EMPTY not allowed for equal position!"); } - const KeyValueType type = firstQe.values.size() ? firstQe.values[0].Type() : KeyValueType::Null{}; - Comparator cmp(firstQe.condition, type, firstQe.values, true, firstQe.distinct, ns.payloadType_, FieldsSet({firstQe.idxNo})); + const KeyValueType type = firstQe.Values().size() ? firstQe.Values()[0].Type() : KeyValueType::Null{}; + Comparator cmp{firstQe.Condition(), type, firstQe.Values(), true, firstQe.Distinct(), ns.payloadType_, firstQe.Fields()}; for (size_t i = 0; i < eqPos.queryEntriesPositions.size(); ++i) { const QueryEntry &qe = queries.Get(eqPos.queryEntriesPositions[i]); - if (qe.condition == CondEmpty || (qe.condition == CondSet && qe.values.empty())) { + if (qe.Condition() == CondEmpty || (qe.Condition() == CondSet && qe.Values().empty())) { throw Error(errLogic, "Condition IN(with empty parameter list), IS NULL, IS EMPTY not allowed for equal position!"); } - if (qe.idxNo == IndexValueType::SetByJsonPath) { - cmp.BindEqualPosition(ns.tagsMatcher_.path2tag(qe.index), qe.values, qe.condition); - } else if (ns.indexes_[qe.idxNo]->Opts().IsSparse()) { - const TagsPath &tp = ns.indexes_[qe.idxNo]->Fields().getTagsPath(0); - cmp.BindEqualPosition(tp, qe.values, qe.condition); + assertrx_throw(qe.Fields().size() == 1); + if (qe.Fields()[0] == IndexValueType::SetByJsonPath) { + cmp.BindEqualPosition(qe.Fields().getFieldsPath(0), qe.Values(), qe.Condition()); } else { - cmp.BindEqualPosition(qe.idxNo, qe.values, qe.condition); + cmp.BindEqualPosition(qe.Fields()[0], qe.Values(), qe.Condition()); } } @@ -450,33 +414,33 @@ std::vector SelectIteratorContainer::pr queries.InvokeAppropriate( j, Skip{}, [&](const QueryEntry &eq) { - if (foundFields.find(eq.index) != foundFields.end()) { + if (foundFields.find(eq.FieldName()) != foundFields.end()) { throw Error(errParams, "Equal position field '%s' found twice in enclosing bracket; equal position fields: [%s]", - eq.index, getEpFieldsStr()); + eq.FieldName(), getEpFieldsStr()); } - const auto it = epFields.find(eq.index); + const auto it = epFields.find(eq.FieldName()); if (it == epFields.end()) return; if (queries.GetOperation(j) != OpAnd || (next < end && queries.GetOperation(next) == OpOr)) { throw Error(errParams, "Only AND operation allowed for equal position; equal position field with not AND operation: '%s'; " "equal position fields: [%s]", - eq.index, getEpFieldsStr()); + eq.FieldName(), getEpFieldsStr()); } result[i].queryEntriesPositions.push_back(j); foundFields.insert(epFields.extract(it)); }, [&](const BetweenFieldsQueryEntry &eq) { // TODO equal positions for BetweenFieldsQueryEntry #1092 - if (epFields.find(eq.firstIndex) != epFields.end()) { + if (epFields.find(eq.LeftFieldName()) != epFields.end()) { throw Error( errParams, "Equal positions for conditions between fields are not supported; field: '%s'; equal position fields: [%s]", - eq.firstIndex, getEpFieldsStr()); + eq.LeftFieldName(), getEpFieldsStr()); } - if (epFields.find(eq.secondIndex) != epFields.end()) { + if (epFields.find(eq.RightFieldName()) != epFields.end()) { throw Error( errParams, "Equal positions for conditions between fields are not supported; field: '%s'; equal position fields: [%s]", - eq.secondIndex, getEpFieldsStr()); + eq.RightFieldName(), getEpFieldsStr()); } }); } @@ -517,28 +481,16 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor &q return contFT; }, [&](const QueryEntry &qe) { - const bool isFT = qe.idxNo != IndexValueType::SetByJsonPath && IsFullText(ns.indexes_[qe.idxNo]->Type()); + const bool isFT = qe.IsFieldIndexed() && IsFullText(ns.indexes_[qe.IndexNo()]->Type()); if (isFT && (op == OpOr || (next < end && queries.GetOperation(next) == OpOr))) { throw Error(errLogic, "OR operation is not allowed with fulltext index"); } SelectKeyResults selectResults; bool isIndexFt = false, isIndexSparse = false; - const bool nonIndexField = (qe.idxNo == IndexValueType::SetByJsonPath); - - if (nonIndexField) { - auto strictMode = ns.config_.strictMode; - if (ctx_) { - if (ctx_->inTransaction) { - strictMode = StrictModeNone; - } else if (ctx_->query.strictMode != StrictModeNotSet) { - strictMode = ctx_->query.strictMode; - } - } - selectResults = processQueryEntry(qe, ns, strictMode); - } else { - bool enableSortIndexOptimize = (ctx_->sortingContext.uncommitedIndex == qe.idxNo) && !sortIndexFound && - (op == OpAnd) && !qe.distinct && (begin == 0) && + if (qe.IsFieldIndexed()) { + bool enableSortIndexOptimize = (ctx_->sortingContext.uncommitedIndex == qe.IndexNo()) && !sortIndexFound && + (op == OpAnd) && !qe.Distinct() && (begin == 0) && (next == end || queries.GetOperation(next) != OpOr); if (enableSortIndexOptimize) { if (!IsExpectingOrderedResults(qe)) { @@ -549,12 +501,22 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor &q } selectResults = processQueryEntry(qe, enableSortIndexOptimize, ns, sortId, isQueryFt, selectFnc, isIndexFt, isIndexSparse, ftCtx, qPreproc, rdxCtx); + } else { + auto strictMode = ns.config_.strictMode; + if (ctx_) { + if (ctx_->inTransaction) { + strictMode = StrictModeNone; + } else if (ctx_->query.strictMode != StrictModeNotSet) { + strictMode = ctx_->query.strictMode; + } + } + selectResults = processQueryEntry(qe, ns, strictMode); } std::optional nextOp; if (next != end) { nextOp = queries.GetOperation(next); } - processQueryEntryResults(selectResults, op, ns, qe, isIndexFt, isIndexSparse, nonIndexField, nextOp); + processQueryEntryResults(selectResults, op, ns, qe, isIndexFt, isIndexSparse, nextOp); if (op != OpOr) { for (auto &ep : equalPositions) { const auto lastPosition = ep.queryEntriesPositions.back(); @@ -574,9 +536,9 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor &q return false; }, [&](const BetweenFieldsQueryEntry &qe) { - FieldsComparator fc{qe.firstIndex, qe.Condition(), qe.secondIndex, ns.payloadType_}; - processField(fc, qe.firstIndex, qe.firstIdxNo, ns); - processField(fc, qe.secondIndex, qe.secondIdxNo, ns); + FieldsComparator fc{qe.LeftFieldName(), qe.Condition(), qe.RightFieldName(), ns.payloadType_}; + processField(fc, qe.LeftFieldData(), ns); + processField(fc, qe.RightFieldData(), ns); Append(op, std::move(fc)); return false; }, @@ -759,11 +721,11 @@ void JoinSelectIterator::Dump(WrSerializer &ser, const std::vector - void processField(FieldsComparator &, std::string_view field, int idxNo, const NamespaceImpl &ns) const; + void processField(FieldsComparator &, const QueryField &, const NamespaceImpl &) const; void processJoinEntry(const JoinQueryEntry &, OpType); void processQueryEntryResults(SelectKeyResults &selectResults, OpType, const NamespaceImpl &ns, const QueryEntry &qe, bool isIndexFt, - bool isIndexSparse, bool nonIndexField, std::optional nextOp); + bool isIndexSparse, std::optional nextOp); struct EqualPositions { h_vector queryEntriesPositions; size_t positionToInsertIterator = 0; diff --git a/cpp_src/core/payload/fieldsset.h b/cpp_src/core/payload/fieldsset.h index 87da7eefe..98c0062ae 100644 --- a/cpp_src/core/payload/fieldsset.h +++ b/cpp_src/core/payload/fieldsset.h @@ -75,18 +75,8 @@ class FieldsSet : protected base_fields_set { } } - void push_back(const TagsPath &tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(tagsPath); - } - } - void push_back(TagsPath &&tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(std::move(tagsPath)); - } - } + void push_back(const TagsPath &tagsPath) { pushBack(tagsPath); } + void push_back(TagsPath &&tagsPath) { pushBack(std::move(tagsPath)); } void push_front(TagsPath &&tagsPath) { if (!contains(tagsPath)) { base_fields_set::insert(begin(), IndexValueType::SetByJsonPath); @@ -94,18 +84,10 @@ class FieldsSet : protected base_fields_set { } } - void push_back(const IndexedTagsPath &tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(tagsPath); - } - } - void push_back(IndexedTagsPath &&tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(std::move(tagsPath)); - } - } + void push_back(const IndexedTagsPath &tagsPath) { pushBack(tagsPath); } + void push_back(IndexedTagsPath &&tagsPath) { pushBack(std::move(tagsPath)); } + void push_back(const FieldsPath &fieldPath) { pushBack(fieldPath); } + void push_back(FieldsPath &&fieldPath) { pushBack(std::move(fieldPath)); } void push_back(int f) { if (f < 0) return; @@ -146,31 +128,34 @@ class FieldsSet : protected base_fields_set { bool contains(const IndexesFieldsSet &f) const noexcept { return (mask_ & f.mask()) == f.mask(); } bool contains(const TagsPath &tagsPath) const noexcept { for (const FieldsPath &path : tagsPaths_) { - if (path.index() == 0) { - if (std::get(path) == tagsPath) return true; - } else { - if (std::get(path).Compare(tagsPath)) return true; + if (std::visit(overloaded{[&tagsPath](const TagsPath &path) { return path == tagsPath; }, + [&tagsPath](const IndexedTagsPath &path) { return path.Compare(tagsPath); }}, + path)) { + return true; } } return false; } bool contains(const IndexedTagsPath &tagsPath) const noexcept { for (const FieldsPath &path : tagsPaths_) { - if (path.index() == 1) { - if (std::get(path) == tagsPath) return true; - } else { - if (tagsPath.Compare(std::get(path))) return true; + if (std::visit(overloaded{[&tagsPath](const TagsPath &path) { return tagsPath.Compare(path); }, + [&tagsPath](const IndexedTagsPath &path) { return path == tagsPath; }}, + path)) { + return true; } } return false; } + bool contains(const FieldsPath &fieldsPath) const noexcept { + return std::visit([&](const auto &fp) { return contains(fp); }, fieldsPath); + } bool match(const TagsPath &tagsPath) const noexcept { if (tagsPaths_.empty()) return true; - for (auto &flt : tagsPaths_) { - if (flt.index() == 0) { - if (comparePaths(tagsPath, std::get(flt))) return true; - } else { - if (comparePaths(std::get(flt), tagsPath)) return true; + for (auto &path : tagsPaths_) { + if (std::visit(overloaded{[&tagsPath, this](const TagsPath &path) { return comparePaths(tagsPath, path); }, + [&tagsPath, this](const IndexedTagsPath &path) { return comparePaths(path, tagsPath); }}, + path)) { + return true; } } return false; @@ -178,11 +163,11 @@ class FieldsSet : protected base_fields_set { template bool match(const IndexedTagsPathImpl &tagsPath) const noexcept { if (tagsPaths_.empty()) return true; - for (auto &flt : tagsPaths_) { - if (flt.index() == 1) { - if (comparePaths(tagsPath, std::get(flt))) return true; - } else { - if (comparePaths(tagsPath, std::get(flt))) return true; + for (auto &path : tagsPaths_) { + if (std::visit(overloaded{[&tagsPath, this](const TagsPath &path) { return comparePaths(tagsPath, path); }, + [&tagsPath, this](const IndexedTagsPath &path) { return comparePaths(tagsPath, path); }}, + path)) { + return true; } } return false; @@ -199,12 +184,15 @@ class FieldsSet : protected base_fields_set { const h_vector &getJsonPaths() const noexcept { return jsonPaths_; } bool isTagsPathIndexed(size_t idx) const noexcept { assertrx(idx < tagsPaths_.size()); - return (tagsPaths_[idx].index() == 1); + return std::visit(overloaded{[](const TagsPath &) { return false; }, [](const IndexedTagsPath &) { return true; }}, + tagsPaths_[idx]); } const TagsPath &getTagsPath(size_t idx) const & { return std::get(tagsPaths_[idx]); } const TagsPath &getTagsPath(size_t idx) const && = delete; const IndexedTagsPath &getIndexedTagsPath(size_t idx) const & { return std::get(tagsPaths_[idx]); } const IndexedTagsPath &getIndexedTagsPath(size_t idx) const && = delete; + const FieldsPath &getFieldsPath(size_t idx) const & { return tagsPaths_[idx]; } + const FieldsPath &getFieldsPath(size_t idx) const && = delete; const std::string &getJsonPath(size_t idx) const &noexcept { return jsonPaths_[idx]; } const std::string &getJsonPath(size_t idx) const && = delete; @@ -235,7 +223,14 @@ class FieldsSet : protected base_fields_set { os << "]}"; } -protected: +private: + template + void pushBack(F &&fieldPath) { + if (!contains(fieldPath)) { + base_fields_set::push_back(IndexValueType::SetByJsonPath); + tagsPaths_.emplace_back(std::forward(fieldPath)); + } + } template bool comparePaths(const TPath1 &lhs, const TPath2 &rhs) const noexcept { unsigned i = 0, count = std::min(lhs.size(), rhs.size()); diff --git a/cpp_src/core/payload/payloadfieldvalue.cc b/cpp_src/core/payload/payloadfieldvalue.cc index e9a0d6a06..2b2defba1 100644 --- a/cpp_src/core/payload/payloadfieldvalue.cc +++ b/cpp_src/core/payload/payloadfieldvalue.cc @@ -3,7 +3,7 @@ namespace reindexer { void PayloadFieldValue::throwSetTypeMissmatch(const Variant& kv) { - throw Error(errLogic, "PayloadFieldValue::Set field '%s' type mismatch. passed '%s', expected '%s'\n", t_.Name(), kv.Type().Name(), + throw Error(errLogic, "PayloadFieldValue::Set field '%s' type mismatch. Passed '%s', expected '%s'", t_.Name(), kv.Type().Name(), t_.Type().Name()); } diff --git a/cpp_src/core/payload/payloadiface.cc b/cpp_src/core/payload/payloadiface.cc index 110d0b80f..64b50daec 100644 --- a/cpp_src/core/payload/payloadiface.cc +++ b/cpp_src/core/payload/payloadiface.cc @@ -5,7 +5,6 @@ #include "core/keyvalue/p_string.h" #include "core/keyvalue/variant.h" #include "core/namespace/stringsholder.h" -#include "itoa/itoa.h" #include "payloadiface.h" #include "payloadvalue.h" @@ -77,29 +76,77 @@ void PayloadIface::GetByJsonPath(std::string_view jsonPath, TagsMatcher &tags } template -void PayloadIface::GetByJsonPath(const TagsPath &jsonPath, VariantArray &krefs, KeyValueType expectedType) const { - ConstPayload pl(t_, *v_); - FieldsSet filter({jsonPath}); - BaseEncoder encoder(nullptr, &filter); +template +void PayloadIface::getByJsonPath(const P &path, VariantArray &krefs, KeyValueType expectedType) const { krefs.clear(); - if (!jsonPath.empty()) { - FieldsExtractor extractor(&krefs, expectedType, jsonPath.size()); - encoder.Encode(pl, extractor); + if (path.empty()) { + return; } + const FieldsSet filter{{path}}; + ConstPayload pl(t_, *v_); + BaseEncoder encoder(nullptr, &filter); + FieldsExtractor extractor(&krefs, expectedType, path.size(), &filter); + encoder.Encode(pl, extractor); +} + +template +void PayloadIface::GetByJsonPath(const TagsPath &tagsPath, VariantArray &krefs, KeyValueType expectedType) const { + getByJsonPath(tagsPath, krefs, expectedType); } template void PayloadIface::GetByJsonPath(const IndexedTagsPath &tagsPath, VariantArray &krefs, KeyValueType expectedType) const { - ConstPayload pl(t_, *v_); - FieldsSet filter({tagsPath}); - BaseEncoder encoder(nullptr, &filter); - krefs.Clear(); - if (!tagsPath.empty()) { - FieldsExtractor extractor(&krefs, expectedType, tagsPath.size(), &filter); - encoder.Encode(pl, extractor); + getByJsonPath(tagsPath, krefs, expectedType); +} + +template +void PayloadIface::GetByFieldsSet(const FieldsSet &fields, VariantArray &kvs, KeyValueType expectedType, + const std::vector &expectedCompositeTypes) const { + if (expectedType.Is()) { + kvs.Clear(); + kvs.emplace_back(GetComposite(fields, expectedCompositeTypes)); + } else { + assertrx_throw(fields.size() == 1); + if (fields[0] == IndexValueType::SetByJsonPath) { + assertrx_throw(fields.getTagsPathsLength() == 1); + if (fields.isTagsPathIndexed(0)) { + getByJsonPath(fields.getIndexedTagsPath(0), kvs, expectedType); + } else { + getByJsonPath(fields.getTagsPath(0), kvs, expectedType); + } + } else { + Get(fields[0], kvs); + } } } +template +Variant PayloadIface::GetComposite(const FieldsSet &fields, const std::vector &expectedTypes) const { + thread_local VariantArray buffer; + buffer.clear(); + assertrx_throw(fields.size() == expectedTypes.size()); + size_t jsonFieldIdx{0}; + [[maybe_unused]] const size_t maxJsonFieldIdx{fields.getTagsPathsLength()}; + VariantArray buf; + for (size_t i = 0, s = fields.size(); i < s; ++i) { + buf.clear(); + if (fields[i] == IndexValueType::SetByJsonPath) { + assertrx_throw(jsonFieldIdx < maxJsonFieldIdx); + if (fields.isTagsPathIndexed(jsonFieldIdx)) { + getByJsonPath(fields.getIndexedTagsPath(jsonFieldIdx), buf, expectedTypes[i]); + } else { + getByJsonPath(fields.getTagsPath(jsonFieldIdx), buf, expectedTypes[i]); + } + ++jsonFieldIdx; + } else { + Get(fields[i], buf); + } + assertrx_throw(buf.size() == 1); + buffer.emplace_back(std::move(buf[0])); + } + return Variant{buffer}; +} + template VariantArray PayloadIface::GetIndexedArrayData(const IndexedTagsPath &tagsPath, int field, int &offset, int &size) const { if (tagsPath.empty()) { @@ -199,8 +246,13 @@ void PayloadIface::SerializeFields(WrSerializer &ser, const FieldsSet &fields for (int field : fields) { if (field == IndexValueType::SetByJsonPath) { assertrx(tagPathIdx < fields.getTagsPathsLength()); - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx); - GetByJsonPath(tagsPath, varr, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx); + GetByJsonPath(tagsPath, varr, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx); + GetByJsonPath(tagsPath, varr, KeyValueType::Undefined{}); + } if (varr.empty()) { throw Error(errParams, "PK serializing error: field [%s] cannot not be empty", fields.getJsonPath(tagPathIdx)); } @@ -281,8 +333,13 @@ size_t PayloadIface::GetHash(const FieldsSet &fields) const { ret ^= Field(field).Hash(); } else { assertrx(tagPathIdx < fields.getTagsPathsLength()); - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + } ret ^= keys1.Hash(); } } @@ -333,9 +390,15 @@ bool PayloadIface::IsEQ(const T &other, const FieldsSet &fields) const { if (!Field(field).IsEQ(o.Field(field))) return false; } } else { - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); - o.GetByJsonPath(tagsPath, keys2, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, keys2, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, keys2, KeyValueType::Undefined{}); + } if (keys1 != keys2) { return false; } @@ -355,35 +418,44 @@ int PayloadIface::Compare(const T &other, const FieldsSet &fields, size_t &fi bool commonOpts = (collateOpts.size() == 1); for (size_t i = 0; i < fields.size(); ++i) { - int cmpRes = 0; const auto field(fields[i]); const CollateOpts *opts(commonOpts ? collateOpts[0] : collateOpts[i]); if (field != IndexValueType::SetByJsonPath) { - cmpRes = Field(field).Get().Compare(o.Field(field).Get(), opts ? *opts : CollateOpts()); + int cmpRes = Field(field).Get().Compare(o.Field(field).Get(), opts ? *opts : CollateOpts()); + if (cmpRes) { + firstDifferentFieldIdx = i; + return cmpRes; + } } else { assertrx(tagPathIdx < fields.getTagsPathsLength()); - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - GetByJsonPath(tagsPath, krefs1, KeyValueType::Undefined{}); - o.GetByJsonPath(tagsPath, krefs2, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, krefs1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, krefs2, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, krefs1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, krefs2, KeyValueType::Undefined{}); + } size_t length = std::min(krefs1.size(), krefs2.size()); for (size_t j = 0; j < length; ++j) { - cmpRes = krefs1[j].RelaxCompare(krefs2[j], opts ? *opts : CollateOpts()); - if (cmpRes) break; - } - if (cmpRes == 0) { - if (krefs1.size() < krefs2.size()) { - cmpRes = -1; - } else if (krefs1.size() > krefs2.size()) { - cmpRes = 1; + int cmpRes = krefs1[j].RelaxCompare(krefs2[j], opts ? *opts : CollateOpts()); + if (cmpRes) { + firstDifferentFieldIdx = i; + return cmpRes; } } - } - - firstDifferentFieldIdx = i; - if (cmpRes > 0) return 1; - if (cmpRes < 0) return -1; + if (krefs1.size() < krefs2.size()) { + firstDifferentFieldIdx = i; + return -1; + } + if (krefs1.size() > krefs2.size()) { + firstDifferentFieldIdx = i; + return 1; + } + } } return 0; } diff --git a/cpp_src/core/payload/payloadiface.h b/cpp_src/core/payload/payloadiface.h index 39569e4c3..1781e1ce9 100644 --- a/cpp_src/core/payload/payloadiface.h +++ b/cpp_src/core/payload/payloadiface.h @@ -112,6 +112,9 @@ class PayloadIface { void GetByJsonPath(std::string_view jsonPath, TagsMatcher &tagsMatcher, VariantArray &, KeyValueType expectedType) const; void GetByJsonPath(const TagsPath &jsonPath, VariantArray &, KeyValueType expectedType) const; void GetByJsonPath(const IndexedTagsPath &jsonPath, VariantArray &, KeyValueType expectedType) const; + void GetByFieldsSet(const FieldsSet &, VariantArray &, KeyValueType expectedType, + const std::vector &expectedCompositeTypes) const; + [[nodiscard]] Variant GetComposite(const FieldsSet &, const std::vector &expectedTypes) const; VariantArray GetIndexedArrayData(const IndexedTagsPath &jsonPath, int field, int &offset, int &size) const; // Get fields count @@ -167,6 +170,8 @@ class PayloadIface { T CopyWithRemovedFields(PayloadType t); template void copyOrMoveStrings(int field, StrHolder &dest, bool copy); + template + void getByJsonPath(const P &path, VariantArray &, KeyValueType expectedType) const; template ::value>::type * = nullptr> void setArray(int field, const VariantArray &keys, bool append); diff --git a/cpp_src/core/payload/payloadtype.h b/cpp_src/core/payload/payloadtype.h index b6cb5a240..96e92f40d 100644 --- a/cpp_src/core/payload/payloadtype.h +++ b/cpp_src/core/payload/payloadtype.h @@ -18,7 +18,7 @@ class PayloadType : public shared_cow_ptr { PayloadType &operator=(PayloadType &&) = default; PayloadType &operator=(const PayloadType &) = default; PayloadType(const std::string &name, std::initializer_list fields = {}); - PayloadType(const PayloadTypeImpl &impl); + explicit PayloadType(const PayloadTypeImpl &impl); ~PayloadType(); const PayloadFieldType &Field(int field) const; diff --git a/cpp_src/core/payload/payloadvalue.cc b/cpp_src/core/payload/payloadvalue.cc index 7e438b0cd..d1b07a365 100644 --- a/cpp_src/core/payload/payloadvalue.cc +++ b/cpp_src/core/payload/payloadvalue.cc @@ -1,27 +1,19 @@ #include "payloadvalue.h" -#include +#include #include "core/keyvalue/p_string.h" -#include "string.h" -#include "tools/errors.h" + namespace reindexer { PayloadValue::PayloadValue(size_t size, const uint8_t *ptr, size_t cap) : p_(nullptr) { p_ = alloc((cap != 0) ? cap : size); - if (ptr) + if (ptr) { memcpy(Ptr(), ptr, size); - else + } else { memset(Ptr(), 0, size); -} - -PayloadValue::PayloadValue(const PayloadValue &other) noexcept : p_(other.p_) { - if (p_) { - header()->refcount.fetch_add(1, std::memory_order_relaxed); } } -PayloadValue::~PayloadValue() { release(); } - uint8_t *PayloadValue::alloc(size_t cap) { auto pn = reinterpret_cast(operator new(cap + sizeof(dataHeader))); dataHeader *nheader = reinterpret_cast(pn); @@ -44,7 +36,7 @@ void PayloadValue::release() noexcept { void PayloadValue::Clone(size_t size) { // If we have exclusive data - just up lsn - if (p_ && header()->refcount.load() == 1) { + if (p_ && header()->refcount.load(std::memory_order_acquire) == 1) { return; } assertrx(size || p_); @@ -64,7 +56,7 @@ void PayloadValue::Clone(size_t size) { void PayloadValue::Resize(size_t oldSize, size_t newSize) { assertrx(p_); - assertrx(header()->refcount.load() == 1); + assertrx(header()->refcount.load(std::memory_order_acquire) == 1); if (newSize <= header()->cap) return; diff --git a/cpp_src/core/payload/payloadvalue.h b/cpp_src/core/payload/payloadvalue.h index 96a53b0b1..b94ba1198 100644 --- a/cpp_src/core/payload/payloadvalue.h +++ b/cpp_src/core/payload/payloadvalue.h @@ -14,18 +14,22 @@ class PayloadValue { struct dataHeader { dataHeader() noexcept : refcount(1), cap(0), lsn(-1) {} - ~dataHeader() { assertrx(refcount.load() == 0); } + ~dataHeader() { assertrx(refcount.load(std::memory_order_acquire) == 0); } refcounter refcount; unsigned cap; int64_t lsn; }; PayloadValue() noexcept : p_(nullptr) {} - PayloadValue(const PayloadValue &) noexcept; + PayloadValue(const PayloadValue &other) noexcept : p_(other.p_) { + if (p_) { + header()->refcount.fetch_add(1, std::memory_order_relaxed); + } + } // Alloc payload store with size, and copy data from another array PayloadValue(size_t size, const uint8_t *ptr = nullptr, size_t cap = 0); - ~PayloadValue(); - PayloadValue &operator=(const PayloadValue &other) { + ~PayloadValue() { release(); } + PayloadValue &operator=(const PayloadValue &other) noexcept { if (&other != this) { release(); p_ = other.p_; @@ -52,8 +56,8 @@ class PayloadValue { uint8_t *Ptr() const noexcept { return p_ + sizeof(dataHeader); } void SetLSN(int64_t lsn) { header()->lsn = lsn; } int64_t GetLSN() const { return p_ ? header()->lsn : 0; } - bool IsFree() const { return bool(p_ == nullptr); } - void Free() { release(); } + bool IsFree() const noexcept { return bool(p_ == nullptr); } + void Free() noexcept { release(); } size_t GetCapacity() const noexcept { return header()->cap; } const uint8_t *get() const noexcept { return p_; } diff --git a/cpp_src/core/query/dsl/dslencoder.cc b/cpp_src/core/query/dsl/dslencoder.cc index 3561d5b24..b8e92f6e3 100644 --- a/cpp_src/core/query/dsl/dslencoder.cc +++ b/cpp_src/core/query/dsl/dslencoder.cc @@ -114,10 +114,10 @@ void encodeAggregationFunctions(const Query& query, JsonBuilder& builder) { } void encodeJoinEntry(const QueryJoinEntry& joinEntry, JsonBuilder& builder) { - builder.Put("left_field", joinEntry.index_); - builder.Put("right_field", joinEntry.joinIndex_); - builder.Put("cond", get(cond_map, joinEntry.condition_)); - builder.Put("op", get(op_map, joinEntry.op_)); + builder.Put("left_field", joinEntry.LeftFieldName()); + builder.Put("right_field", joinEntry.RightFieldName()); + builder.Put("cond", get(cond_map, joinEntry.Condition())); + builder.Put("op", get(op_map, joinEntry.Operation())); } void encodeSingleJoinQuery(const JoinedQuery& joinQuery, JsonBuilder& builder) { @@ -125,9 +125,9 @@ void encodeSingleJoinQuery(const JoinedQuery& joinQuery, JsonBuilder& builder) { auto node = builder.Object("join_query"sv); node.Put("type", get(join_types, joinQuery.joinType)); - node.Put("namespace", joinQuery._namespace); - node.Put("limit", joinQuery.count); - node.Put("offset", joinQuery.start); + node.Put("namespace", joinQuery.NsName()); + node.Put("limit", joinQuery.Limit()); + node.Put("offset", joinQuery.Offset()); encodeFilters(joinQuery, node); encodeSorting(joinQuery.sortingEntries_, node); @@ -147,18 +147,18 @@ void encodeSingleJoinQuery(const JoinedQuery& joinQuery, JsonBuilder& builder) { } void encodeFilter(const QueryEntry& qentry, JsonBuilder& builder) { - if (qentry.distinct) return; - builder.Put("cond", get(cond_map, CondType(qentry.condition))); - builder.Put("field", qentry.index); + if (qentry.Distinct()) return; + builder.Put("cond", get(cond_map, CondType(qentry.Condition()))); + builder.Put("field", qentry.FieldName()); - if (qentry.values.empty()) return; - if (qentry.values.size() > 1 || qentry.values[0].Type().Is()) { + if (qentry.Values().empty()) return; + if (qentry.Values().size() > 1 || qentry.Values()[0].Type().Is()) { auto arrNode = builder.Array("value"); - for (const Variant& kv : qentry.values) { + for (const Variant& kv : qentry.Values()) { arrNode.Put(nullptr, kv); } } else { - builder.Put("value", qentry.values[0]); + builder.Put("value", qentry.Values()[0]); } } @@ -201,10 +201,10 @@ void encodeUpdateFields(const Query& query, JsonBuilder& builder) { void toDsl(const Query& query, JsonBuilder& builder) { switch (query.Type()) { case QueryType::QuerySelect: { - builder.Put("namespace", query._namespace); - builder.Put("limit", query.count); - builder.Put("offset", query.start); - builder.Put("req_total", get(reqtotal_values, query.calcTotal)); + builder.Put("namespace", query.NsName()); + builder.Put("limit", query.Limit()); + builder.Put("offset", query.Offset()); + builder.Put("req_total", get(reqtotal_values, query.CalcTotal())); builder.Put("explain", query.explain_); builder.Put("type", "select"); auto strictMode = strictModeToString(query.strictMode); @@ -222,7 +222,7 @@ void toDsl(const Query& query, JsonBuilder& builder) { break; } case QueryType::QueryUpdate: { - builder.Put("namespace", query._namespace); + builder.Put("namespace", query.NsName()); builder.Put("explain", query.explain_); builder.Put("type", "update"); encodeFilters(query, builder); @@ -244,14 +244,14 @@ void toDsl(const Query& query, JsonBuilder& builder) { break; } case QueryType::QueryDelete: { - builder.Put("namespace", query._namespace); + builder.Put("namespace", query.NsName()); builder.Put("explain", query.explain_); builder.Put("type", "delete"); encodeFilters(query, builder); break; } case QueryType::QueryTruncate: { - builder.Put("namespace", query._namespace); + builder.Put("namespace", query.NsName()); builder.Put("type", "truncate"); break; } @@ -283,7 +283,7 @@ void QueryEntries::toDsl(const_iterator it, const_iterator to, const Query& pare dsl::encodeEqualPositions(bracket.equalPositions, arrNode); }, [&node](const QueryEntry& qe) { - if (qe.distinct) return; + if (qe.Distinct()) return; dsl::encodeFilter(qe, node); }, [&node, &parentQuery](const JoinQueryEntry& jqe) { @@ -292,8 +292,8 @@ void QueryEntries::toDsl(const_iterator it, const_iterator to, const Query& pare }, [&node](const BetweenFieldsQueryEntry& qe) { node.Put("cond", dsl::get(dsl::cond_map, CondType(qe.Condition()))); - node.Put("first_field", qe.firstIndex); - node.Put("second_field", qe.secondIndex); + node.Put("first_field", qe.LeftFieldName()); + node.Put("second_field", qe.RightFieldName()); }); } } diff --git a/cpp_src/core/query/dsl/dslparser.cc b/cpp_src/core/query/dsl/dslparser.cc index f5fbfc709..13bac0c61 100644 --- a/cpp_src/core/query/dsl/dslparser.cc +++ b/cpp_src/core/query/dsl/dslparser.cc @@ -1,6 +1,5 @@ #include "dslparser.h" #include "core/cjson/jschemachecker.h" -#include "core/cjson/jsonbuilder.h" #include "core/query/query.h" #include "estl/fast_hash_map.h" #include "gason/gason.h" @@ -324,34 +323,6 @@ static void parseFilter(const JsonValue& filter, Query& q, std::vector(joined_entry_map, name, "join_query.on"sv)) { case JoinEntry::LeftField: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.index_ = std::string(value.toString()); + leftField = std::string(value.toString()); break; case JoinEntry::RightField: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.joinIndex_ = std::string(value.toString()); + rightField = std::string(value.toString()); break; case JoinEntry::Cond: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.condition_ = get(cond_map, value.toString(), "condition enum"sv); + cond = get(cond_map, value.toString(), "condition enum"sv); break; case JoinEntry::Op: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.op_ = get(op_map, value.toString(), "operation enum"sv); + op = get(op_map, value.toString(), "operation enum"sv); break; } } - qjoin.joinEntries_.emplace_back(qjoinEntry); + qjoin.joinEntries_.emplace_back(op, cond, std::move(leftField), std::move(rightField)); } } @@ -419,7 +392,7 @@ void parseSingleJoinQuery(const JsonValue& join, Query& query) { break; case JoinRoot::Namespace: checkJsonValueType(value, name, JSON_STRING); - qjoin._namespace = std::string(value.toString()); + qjoin.SetNsName(value.toString()); break; case JoinRoot::Filters: checkJsonValueType(value, name, JSON_ARRAY); @@ -430,11 +403,11 @@ void parseSingleJoinQuery(const JsonValue& join, Query& query) { break; case JoinRoot::Limit: checkJsonValueType(value, name, JSON_NUMBER, JSON_DOUBLE); - qjoin.count = static_cast(value.toNumber()); + qjoin.Limit(static_cast(value.toNumber())); break; case JoinRoot::Offset: checkJsonValueType(value, name, JSON_NUMBER, JSON_DOUBLE); - qjoin.start = static_cast(value.toNumber()); + qjoin.Offset(static_cast(value.toNumber())); break; case JoinRoot::On: parseJoinedEntries(value, qjoin); @@ -604,17 +577,17 @@ void parse(const JsonValue& root, Query& q) { switch (get(root_map, name, "root"sv)) { case Root::Namespace: checkJsonValueType(v, name, JSON_STRING); - q._namespace = std::string(v.toString()); + q.SetNsName(v.toString()); break; case Root::Limit: checkJsonValueType(v, name, JSON_NUMBER, JSON_DOUBLE); - q.count = static_cast(v.toNumber()); + q.Limit(static_cast(v.toNumber())); break; case Root::Offset: checkJsonValueType(v, name, JSON_NUMBER, JSON_DOUBLE); - q.start = static_cast(v.toNumber()); + q.Offset(static_cast(v.toNumber())); break; case Root::Filters: @@ -643,7 +616,7 @@ void parse(const JsonValue& root, Query& q) { break; case Root::ReqTotal: checkJsonValueType(v, name, JSON_STRING); - q.calcTotal = get(reqtotal_values, v.toString(), "req_total enum"sv); + q.CalcTotal(get(reqtotal_values, v.toString(), "req_total enum"sv)); break; case Root::Aggregations: checkJsonValueType(v, name, JSON_ARRAY); diff --git a/cpp_src/core/query/query.cc b/cpp_src/core/query/query.cc index 2e7b1da3e..49b6f3dec 100644 --- a/cpp_src/core/query/query.cc +++ b/cpp_src/core/query/query.cc @@ -13,18 +13,15 @@ using namespace std::string_view_literals; const std::string_view kLsnIndexName = "#lsn"sv; const std::string_view kSlaveVersionIndexName = "#slave_version"sv; -Query::Query(const std::string &__namespace, unsigned _start, unsigned _count, CalcTotalMode _calcTotal) - : _namespace(__namespace), start(_start), count(_count), calcTotal(_calcTotal) {} - bool Query::operator==(const Query &obj) const { if (entries != obj.entries) return false; if (aggregations_ != obj.aggregations_) return false; - if (_namespace != obj._namespace) return false; + if (NsName() != obj.NsName()) return false; if (sortingEntries_ != obj.sortingEntries_) return false; - if (calcTotal != obj.calcTotal) return false; - if (start != obj.start) return false; - if (count != obj.count) return false; + if (CalcTotal() != obj.CalcTotal()) return false; + if (Offset() != obj.Offset()) return false; + if (Limit() != obj.Limit()) return false; if (debugLevel != obj.debugLevel) return false; if (strictMode != obj.strictMode) return false; if (forcedSortOrder_.size() != obj.forcedSortOrder_.size()) return false; @@ -52,17 +49,6 @@ Error Query::FromJSON(const std::string &dsl) { return dsl::Parse(dsl, *this); } std::string Query::GetJSON() const { return dsl::toDsl(*this); } -Query &Query::SetObject(std::string field, VariantArray value, bool hasExpressions) & { - for (auto &it : value) { - if (!it.Type().Is()) { - throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", - it.Type().Name()); - } - } - updateFields_.emplace_back(std::move(field), std::move(value), FieldModeSetJson, hasExpressions); - return *this; -} - WrSerializer &Query::GetSQL(WrSerializer &ser, bool stripArgs) const { return SQLEncoder(*this).GetSQL(ser, stripArgs); } std::string Query::GetSQL(bool stripArgs) const { @@ -70,6 +56,11 @@ std::string Query::GetSQL(bool stripArgs) const { return std::string(GetSQL(ser, stripArgs).Slice()); } +std::string Query::GetSQL(QueryType realType) const { + WrSerializer ser; + return std::string(SQLEncoder(*this, realType).GetSQL(ser, false).Slice()); +} + void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { bool end = false; std::vector> equalPositions; @@ -77,12 +68,12 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { int qtype = ser.GetVarUint(); switch (qtype) { case QueryCondition: { - QueryEntry qe; - qe.index = std::string(ser.GetVString()); - OpType op = OpType(ser.GetVarUint()); - qe.condition = CondType(ser.GetVarUint()); + const auto fieldName = ser.GetVString(); + const OpType op = OpType(ser.GetVarUint()); + const CondType condition = CondType(ser.GetVarUint()); int cnt = ser.GetVarUint(); - if (qe.condition == CondDWithin) { + VariantArray values; + if (condition == CondDWithin) { if (cnt != 3) { throw Error(errParseBin, "Expected point and distance for DWithin"); } @@ -90,14 +81,14 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { point.reserve(2); point.emplace_back(ser.GetVariant().EnsureHold()); point.emplace_back(ser.GetVariant().EnsureHold()); - qe.values.reserve(2); - qe.values.emplace_back(std::move(point)); - qe.values.emplace_back(ser.GetVariant().EnsureHold()); + values.reserve(2); + values.emplace_back(std::move(point)); + values.emplace_back(ser.GetVariant().EnsureHold()); } else { - qe.values.reserve(cnt); - while (cnt--) qe.values.emplace_back(ser.GetVariant().EnsureHold()); + values.reserve(cnt); + while (cnt--) values.emplace_back(ser.GetVariant().EnsureHold()); } - entries.Append(op, std::move(qe)); + entries.Append(op, std::string{fieldName}, condition, std::move(values)); break; } case QueryBetweenFieldsCondition: { @@ -105,12 +96,12 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { std::string firstField{ser.GetVString()}; CondType condition = static_cast(ser.GetVarUint()); std::string secondField{ser.GetVString()}; - entries.Append(op, BetweenFieldsQueryEntry{std::move(firstField), condition, std::move(secondField)}); + entries.Append(op, std::move(firstField), condition, std::move(secondField)); break; } case QueryAlwaysFalseCondition: { const OpType op = OpType(ser.GetVarUint()); - entries.Append(op, AlwaysFalse{}); + entries.Append(op); break; } case QueryJoinCondition: { @@ -155,12 +146,9 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { break; } case QueryDistinct: { - QueryEntry qe; - qe.index = std::string(ser.GetVString()); - if (!qe.index.empty()) { - qe.distinct = true; - qe.condition = CondAny; - entries.Append(OpAnd, std::move(qe)); + const auto fieldName = ser.GetVString(); + if (!fieldName.empty()) { + entries.Append(OpAnd, std::string{fieldName}, QueryEntry::DistinctTag{}); } break; } @@ -180,12 +168,12 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { break; } case QueryJoinOn: { - QueryJoinEntry qje; - qje.op_ = OpType(ser.GetVarUint()); - qje.condition_ = CondType(ser.GetVarUint()); - qje.index_ = std::string(ser.GetVString()); - qje.joinIndex_ = std::string(ser.GetVString()); - reinterpret_cast(this)->joinEntries_.push_back(std::move(qje)); + const OpType op = static_cast(ser.GetVarUint()); + const CondType condition = static_cast(ser.GetVarUint()); + std::string leftFieldName{ser.GetVString()}; + std::string rightFieldName{ser.GetVString()}; + reinterpret_cast(this)->joinEntries_.emplace_back(op, condition, std::move(leftFieldName), + std::move(rightFieldName)); break; } case QueryDebugLevel: @@ -195,13 +183,13 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { strictMode = StrictMode(ser.GetVarUint()); break; case QueryLimit: - count = ser.GetVarUint(); + count_ = ser.GetVarUint(); break; case QueryOffset: - start = ser.GetVarUint(); + start_ = ser.GetVarUint(); break; case QueryReqTotal: - calcTotal = CalcTotalMode(ser.GetVarUint()); + calcTotal_ = CalcTotalMode(ser.GetVarUint()); break; case QuerySelectFilter: selectFilter_.push_back(std::string(ser.GetVString())); @@ -291,7 +279,7 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { } void Query::Serialize(WrSerializer &ser, uint8_t mode) const { - ser.PutVString(_namespace); + ser.PutVString(NsName()); entries.Serialize(ser); for (const auto &agg : aggregations_) { @@ -328,10 +316,10 @@ void Query::Serialize(WrSerializer &ser, uint8_t mode) const { if (mode & WithJoinEntries) { for (const auto &qje : reinterpret_cast(this)->joinEntries_) { ser.PutVarUint(QueryJoinOn); - ser.PutVarUint(qje.op_); - ser.PutVarUint(qje.condition_); - ser.PutVString(qje.index_); - ser.PutVString(qje.joinIndex_); + ser.PutVarUint(qje.Operation()); + ser.PutVarUint(qje.Condition()); + ser.PutVString(qje.LeftFieldName()); + ser.PutVString(qje.RightFieldName()); } } @@ -364,17 +352,17 @@ void Query::Serialize(WrSerializer &ser, uint8_t mode) const { if (!(mode & SkipLimitOffset)) { if (HasLimit()) { ser.PutVarUint(QueryLimit); - ser.PutVarUint(count); + ser.PutVarUint(Limit()); } if (HasOffset()) { ser.PutVarUint(QueryOffset); - ser.PutVarUint(start); + ser.PutVarUint(Offset()); } } - if (calcTotal != ModeNoTotal) { + if (CalcTotal() != ModeNoTotal) { ser.PutVarUint(QueryReqTotal); - ser.PutVarUint(calcTotal); + ser.PutVarUint(CalcTotal()); } for (const auto &sf : selectFilter_) { @@ -426,7 +414,7 @@ void Query::Serialize(WrSerializer &ser, uint8_t mode) const { } void Query::Deserialize(Serializer &ser) { - _namespace = std::string(ser.GetVString()); + namespace_ = std::string(ser.GetVString()); bool hasJoinConditions = false; deserialize(ser, hasJoinConditions); @@ -445,37 +433,27 @@ void Query::Deserialize(Serializer &ser) { Query &q = nested ? mergeQueries_.back() : *this; if (joinType != JoinType::LeftJoin && !hasJoinConditions) { const size_t joinIdx = joinQueries_.size(); - entries.Append((joinType == JoinType::OrInnerJoin) ? OpOr : OpAnd, JoinQueryEntry{joinIdx}); + entries.Append((joinType == JoinType::OrInnerJoin) ? OpOr : OpAnd, joinIdx); } q.joinQueries_.emplace_back(std::move(q1)); } } } -Query &Query::Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, Query &&qr) & { - QueryJoinEntry joinEntry; - joinEntry.op_ = op; - joinEntry.condition_ = cond; - joinEntry.index_ = index; - joinEntry.joinIndex_ = joinIndex; +Query &Query::Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, Query &&qr) & { auto &jq = joinQueries_.emplace_back(joinType, std::move(qr)); - jq.joinEntries_.emplace_back(std::move(joinEntry)); + jq.joinEntries_.emplace_back(op, cond, std::move(leftField), std::move(rightField)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return *this; } -Query &Query::Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, const Query &qr) & { - QueryJoinEntry joinEntry; - joinEntry.op_ = op; - joinEntry.condition_ = cond; - joinEntry.index_ = index; - joinEntry.joinIndex_ = joinIndex; +Query &Query::Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, const Query &qr) & { joinQueries_.emplace_back(joinType, qr); - joinQueries_.back().joinEntries_.emplace_back(std::move(joinEntry)); + joinQueries_.back().joinEntries_.emplace_back(op, cond, std::move(leftField), std::move(rightField)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return *this; } @@ -483,7 +461,7 @@ Query &Query::Join(JoinType joinType, const std::string &index, const std::strin Query::OnHelper Query::Join(JoinType joinType, Query &&q) & { joinQueries_.emplace_back(joinType, std::move(q)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {*this, joinQueries_.back()}; } @@ -491,7 +469,7 @@ Query::OnHelper Query::Join(JoinType joinType, Query &&q) & { Query::OnHelper Query::Join(JoinType joinType, const Query &q) & { joinQueries_.emplace_back(joinType, q); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {*this, joinQueries_.back()}; } @@ -499,7 +477,7 @@ Query::OnHelper Query::Join(JoinType joinType, const Query &q) & { Query::OnHelperR Query::Join(JoinType joinType, Query &&q) && { joinQueries_.emplace_back(joinType, std::move(q)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {std::move(*this), joinQueries_.back()}; } @@ -507,7 +485,7 @@ Query::OnHelperR Query::Join(JoinType joinType, Query &&q) && { Query::OnHelperR Query::Join(JoinType joinType, const Query &q) && { joinQueries_.emplace_back(joinType, q); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {std::move(*this), joinQueries_.back()}; } @@ -548,11 +526,11 @@ void Query::WalkNested(bool withSelf, bool withMerged, const std::function(0) && kLsnIndexName == entries.Get(0).index) { + if (entries.Size() == 1 && entries.HoldsOrReferTo(0) && kLsnIndexName == entries.Get(0).FieldName()) { return true; } else if (entries.Size() == 2 && entries.HoldsOrReferTo(0) && entries.HoldsOrReferTo(1)) { - const auto &index0 = entries.Get(0).index; - const auto &index1 = entries.Get(1).index; + const auto &index0 = entries.Get(0).FieldName(); + const auto &index1 = entries.Get(1).FieldName(); return (kLsnIndexName == index0 && kSlaveVersionIndexName == index1) || (kLsnIndexName == index1 && kSlaveVersionIndexName == index0); } diff --git a/cpp_src/core/query/query.h b/cpp_src/core/query/query.h index 00b2b7203..355cd805d 100644 --- a/cpp_src/core/query/query.h +++ b/cpp_src/core/query/query.h @@ -71,8 +71,10 @@ class Query { /// @param start - number of the first row to get from selected set. Analog to sql OFFSET Offset. /// @param count - number of rows to get from result set. Analog to sql LIMIT RowsCount. /// @param calcTotal - calculation mode. - explicit Query(const std::string &nsName, unsigned start = QueryEntry::kDefaultOffset, unsigned count = QueryEntry::kDefaultLimit, - CalcTotalMode calcTotal = ModeNoTotal); + template + explicit Query(T nsName, unsigned start = QueryEntry::kDefaultOffset, unsigned count = QueryEntry::kDefaultLimit, + CalcTotalMode calcTotal = ModeNoTotal) + : namespace_(std::forward(nsName)), start_(start), count_(count), calcTotal_(calcTotal) {} /// Creates an empty object. Query() = default; @@ -94,6 +96,11 @@ class Query { /// @return Query in SQL format std::string GetSQL(bool stripArgs = false) const; + /// Logs query in 'Select field1, ... field N from namespace ...' format. + /// @param realType - replaces original query's type + /// @return Query in SQL format + std::string GetSQL(QueryType realType) const; + /// Parses JSON dsl set. /// @param dsl - dsl set. /// @return always returns errOk or throws an exception. @@ -112,79 +119,75 @@ class Query { Query &&Explain(bool on = true) && { return std::move(Explain(on)); } /// Adds a condition with a single value. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param val - value of index to be compared with. /// @return Query object ready to be executed. - template - Query &Where(const std::string &idx, CondType cond, Input val) & { - return Where(idx, cond, {val}); + template + Query &Where(Str &&field, CondType cond, Input val) & { + return Where(std::forward(field), cond, {std::forward(val)}); } - template - Query &&Where(const std::string &idx, CondType cond, Input val) && { - return std::move(Where(idx, cond, {val})); + template + Query &&Where(Str &&field, CondType cond, Input val) && { + return std::move(Where(std::forward(field), cond, {std::move(val)})); } /// Adds a condition with several values. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param l - list of index values to be compared with. /// @return Query object ready to be executed. - template - Query &Where(const std::string &idx, CondType cond, std::initializer_list l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - for (auto it = l.begin(); it != l.end(); it++) qe.values.push_back(Variant(*it)); - entries.Append(nextOp_, std::move(qe)); + template + Query &Where(Str &&field, CondType cond, std::initializer_list l) & { + VariantArray values; + values.reserve(l.size()); + for (auto it = l.begin(); it != l.end(); it++) values.emplace_back(*it); + entries.Append(nextOp_, std::forward(field), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - template - Query &&Where(const std::string &idx, CondType cond, std::initializer_list l) && { - return std::move(Where(idx, cond, std::move(l))); + template + Query &&Where(Str &&field, CondType cond, std::initializer_list l) && { + return std::move(Where(std::forward(field), cond, std::move(l))); } /// Adds a condition with several values. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param l - vector of index values to be compared with. /// @return Query object ready to be executed. - template - Query &Where(const std::string &idx, CondType cond, const std::vector &l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(l.size()); - for (auto it = l.begin(); it != l.end(); it++) qe.values.push_back(Variant(*it)); - entries.Append(nextOp_, std::move(qe)); + template + Query &Where(Str &&field, CondType cond, const std::vector &l) & { + VariantArray values; + values.reserve(l.size()); + for (auto it = l.begin(); it != l.end(); it++) values.emplace_back(*it); + entries.Append(nextOp_, std::forward(field), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - template - Query &&Where(const std::string &idx, CondType cond, const std::vector &l) && { - return std::move(Where(idx, cond, l)); + template + Query &&Where(Str &&field, CondType cond, const std::vector &l) && { + return std::move(Where(std::forward(field), cond, l)); } /// Adds a condition with several values. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param l - vector of index values to be compared with. /// @return Query object ready to be executed. - Query &Where(const std::string &idx, CondType cond, const VariantArray &l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(l.size()); - for (auto it = l.begin(); it != l.end(); it++) qe.values.push_back(Variant(*it)); - entries.Append(nextOp_, std::move(qe)); + template + Query &Where(Str &&field, CondType cond, VariantArray l) & { + entries.Append(nextOp_, std::forward(field), cond, std::move(l)); nextOp_ = OpAnd; return *this; } - Query &&Where(const std::string &idx, CondType cond, const VariantArray &l) && { return std::move(Where(idx, cond, l)); } + template + Query &&Where(Str &&field, CondType cond, VariantArray l) && { + return std::move(Where(std::forward(field), cond, std::move(l))); + } /// Adds a condition with several values to a composite index. - /// @param idx - index name. + /// @param idx - composite index name. /// @param cond - type of condition. /// @param l - list of values to be compared according to the order /// of indexes in composite index name. @@ -194,172 +197,192 @@ class Query { /// in case of CondRange) belongs to "bookid" and l[0][1] (and l[1][1] in case of CondRange) /// belongs to "price" indexes. /// @return Query object ready to be executed. - Query &WhereComposite(const std::string &idx, CondType cond, std::initializer_list l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(l.size()); + template + Query &WhereComposite(Str &&idx, CondType cond, std::initializer_list l) & { + VariantArray values; + values.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) { - qe.values.push_back(Variant(*it)); + values.emplace_back(*it); } - entries.Append(nextOp_, std::move(qe)); + entries.Append(nextOp_, std::forward(idx), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - Query &&WhereComposite(const std::string &idx, CondType cond, std::initializer_list l) && { - return std::move(WhereComposite(idx, cond, l)); + template + Query &&WhereComposite(Str &&idx, CondType cond, std::initializer_list l) && { + return std::move(WhereComposite(std::forward(idx), cond, std::move(l))); } - Query &WhereComposite(const std::string &idx, CondType cond, const std::vector &v) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(v.size()); + template + Query &WhereComposite(Str &&idx, CondType cond, const std::vector &v) & { + VariantArray values; + values.reserve(v.size()); for (auto it = v.begin(); it != v.end(); it++) { - qe.values.push_back(Variant(*it)); + values.emplace_back(*it); } - entries.Append(nextOp_, std::move(qe)); + entries.Append(nextOp_, std::forward(idx), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - Query &&WhereComposite(const std::string &idx, CondType cond, const std::vector &v) && { - return std::move(WhereComposite(idx, cond, v)); + template + Query &&WhereComposite(Str &&idx, CondType cond, const std::vector &v) && { + return std::move(WhereComposite(std::forward(idx), cond, v)); } - Query &WhereBetweenFields(std::string firstIdx, CondType cond, std::string secondIdx) & { - entries.Append(nextOp_, BetweenFieldsQueryEntry{std::move(firstIdx), cond, std::move(secondIdx)}); + template + Query &WhereBetweenFields(Str1 &&firstIdx, CondType cond, Str2 &&secondIdx) & { + entries.Append(nextOp_, std::forward(firstIdx), cond, std::forward(secondIdx)); nextOp_ = OpAnd; return *this; } - Query &&WhereBetweenFields(std::string firstIdx, CondType cond, std::string secondIdx) && { - return std::move(WhereBetweenFields(std::move(firstIdx), cond, std::move(secondIdx))); + template + Query &&WhereBetweenFields(Str1 &&firstIdx, CondType cond, Str2 &&secondIdx) && { + return std::move(WhereBetweenFields(std::forward(firstIdx), cond, std::forward(secondIdx))); } - Query &DWithin(const std::string &idx, Point p, double distance) & { - QueryEntry qe; - qe.condition = CondDWithin; - qe.index = idx; - qe.values.reserve(2); - qe.values.emplace_back(p); - qe.values.emplace_back(distance); - entries.Append(nextOp_, std::move(qe)); + template + Query &DWithin(Str &&field, Point p, double distance) & { + entries.Append(nextOp_, std::forward(field), CondDWithin, VariantArray::Create(p, distance)); nextOp_ = OpAnd; return *this; } - Query &&DWithin(const std::string &idx, Point p, double distance) && { return std::move(DWithin(idx, p, distance)); } + template + Query &&DWithin(Str &&field, Point p, double distance) && { + return std::move(DWithin(std::forward(field), p, distance)); + } /// Sets a new value for a field. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &Set(std::string field, ValueType value, bool hasExpressions = false) & { - return Set(std::move(field), {value}, hasExpressions); + template + Query &Set(Str &&field, ValueType value, bool hasExpressions = false) & { + return Set(std::forward(field), {value}, hasExpressions); } - template - Query &&Set(std::string field, ValueType value, bool hasExpressions = false) && { - return std::move(Set(std::move(field), std::move(value), hasExpressions)); + template + Query &&Set(Str &&field, ValueType value, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), std::move(value), hasExpressions)); } /// Sets a new value for a field. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &Set(std::string field, std::initializer_list l, bool hasExpressions = false) & { + template + Query &Set(Str &&field, std::initializer_list l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(*it); - return Set(std::move(field), std::move(value), hasExpressions); + return Set(std::forward(field), std::move(value), hasExpressions); } - template - Query &&Set(std::string field, std::initializer_list l, bool hasExpressions = false) && { - return std::move(Set(std::move(field), std::move(l), hasExpressions)); + template + Query &&Set(Str &&field, std::initializer_list l, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), std::move(l), hasExpressions)); } /// Sets a new value for a field. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &Set(std::string field, const std::vector &l, bool hasExpressions = false) & { + template + Query &Set(Str &&field, const std::vector &l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(*it); - return Set(std::move(field), std::move(value.MarkArray()), hasExpressions); + return Set(std::forward(field), std::move(value.MarkArray()), hasExpressions); } - template - Query &&Set(std::string field, const std::vector &l, bool hasExpressions = false) && { - return std::move(Set(std::move(field), l, hasExpressions)); + template + Query &&Set(Str &&field, const std::vector &l, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), l, hasExpressions)); } /// Sets a new value for a field. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - Query &Set(std::string field, VariantArray value, bool hasExpressions = false) & { - updateFields_.emplace_back(std::move(field), std::move(value), FieldModeSet, hasExpressions); + template + Query &Set(Str &&field, VariantArray value, bool hasExpressions = false) & { + updateFields_.emplace_back(std::forward(field), std::move(value), FieldModeSet, hasExpressions); return *this; } - Query &&Set(std::string field, VariantArray value, bool hasExpressions = false) && { - return std::move(Set(std::move(field), std::move(value), hasExpressions)); + template + Query &&Set(Str &&field, VariantArray value, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), std::move(value), hasExpressions)); } /// Sets a value for a field as an object. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &SetObject(std::string field, ValueType value, bool hasExpressions = false) & { - return SetObject(std::move(field), {value}, hasExpressions); + template + Query &SetObject(Str &&field, ValueType value, bool hasExpressions = false) & { + return SetObject(std::forward(field), {value}, hasExpressions); } - template - Query &&SetObject(std::string field, ValueType value, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), std::move(value), hasExpressions)); + template + Query &&SetObject(Str &&field, ValueType value, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), std::move(value), hasExpressions)); } /// Sets a new value for a field as an object. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &SetObject(std::string field, std::initializer_list l, bool hasExpressions = false) & { + template + Query &SetObject(Str &&field, std::initializer_list l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); - for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(Variant(*it)); - return SetObject(std::move(field), std::move(value), hasExpressions); + for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(*it); + return SetObject(std::forward(field), std::move(value), hasExpressions); } - template - Query &&SetObject(std::string field, std::initializer_list l, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), std::move(l), hasExpressions)); + template + Query &&SetObject(Str &&field, std::initializer_list l, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), std::move(l), hasExpressions)); } /// Sets a new value for a field as an object. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &SetObject(std::string field, const std::vector &l, bool hasExpressions = false) & { + template + Query &SetObject(Str &&field, const std::vector &l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(Variant(*it)); - return SetObject(std::move(field), std::move(value.MarkArray()), hasExpressions); + return SetObject(std::forward(field), std::move(value.MarkArray()), hasExpressions); } - template - Query &&SetObject(std::string field, const std::vector &l, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), l, hasExpressions)); + template + Query &&SetObject(Str &&field, const std::vector &l, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), l, hasExpressions)); } /// Sets a value for a field as an object. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - Query &SetObject(std::string field, VariantArray value, bool hasExpressions = false) &; - Query &&SetObject(std::string field, VariantArray value, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), std::move(value), hasExpressions)); + template + Query &SetObject(Str &&field, VariantArray value, bool hasExpressions = false) & { + for (auto &it : value) { + if (!it.Type().Is()) { + throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", + it.Type().Name()); + } + } + updateFields_.emplace_back(std::forward(field), std::move(value), FieldModeSetJson, hasExpressions); + return *this; + } + template + Query &&SetObject(Str &&field, VariantArray value, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), std::move(value), hasExpressions)); } /// Drops a value for a field. /// @param field - field name. - Query &Drop(std::string field) & { - updateFields_.emplace_back(std::move(field), VariantArray(), FieldModeDrop); + template + Query &Drop(Str &&field) & { + updateFields_.emplace_back(std::forward(field), VariantArray(), FieldModeDrop); return *this; } - Query &&Drop(std::string field) && { return std::move(Drop(std::move(field))); } + template + Query &&Drop(Str &&field) && { + return std::move(Drop(std::forward(field))); + } /// Add sql-function to query. /// @param function - function declaration. - void AddFunction(std::string function) { selectFunctions_.emplace_back(std::move(function)); } + template + void AddFunction(Str &&function) { + selectFunctions_.emplace_back(std::forward(function)); + } /// Adds equal position fields to arrays queries. /// @param equalPosition - list of fields with equal array index position. @@ -387,19 +410,21 @@ class Query { /// Joins namespace with another namespace. Analog to sql JOIN. /// @param joinType - type of Join (Inner, Left or OrInner). - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param op - operation type (and, or, not). /// @param qr - query of the namespace that is going to be joined with this one. /// @return Query object ready to be executed. - Query &Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, Query &&qr) &; - Query &&Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, Query &&qr) && { - return std::move(Join(joinType, index, joinIndex, cond, op, std::move(qr))); + Query &Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, Query &&qr) &; + template + Query &&Join(JoinType joinType, StrL &&leftField, StrR &&rightField, CondType cond, OpType op, Query &&qr) && { + return std::move(Join(joinType, std::forward(leftField), std::forward(rightField), cond, op, std::move(qr))); } - Query &Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, const Query &qr) &; - Query &&Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, const Query &qr) && { - return std::move(Join(joinType, index, joinIndex, cond, op, qr)); + Query &Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, const Query &qr) &; + template + Query &&Join(JoinType joinType, StrL &&leftField, StrR &&rightField, CondType cond, OpType op, const Query &qr) && { + return std::move(Join(joinType, std::forward(leftField), std::forward(rightField), cond, op, qr)); } OnHelper Join(JoinType joinType, Query &&q) &; @@ -409,60 +434,72 @@ class Query { /// @public /// Inner Join of this namespace with another one. - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param qr - query of the namespace that is going to be joined with this one. /// @return Query object ready to be executed. - Query &InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) & { // -V1071 - return Join(JoinType::InnerJoin, index, joinIndex, cond, OpAnd, std::move(qr)); + template + Query &InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) & { // -V1071 + return Join(JoinType::InnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, std::move(qr)); } - Query &&InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) && { - return std::move(InnerJoin(index, joinIndex, cond, std::move(qr))); + template + Query &&InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) && { + return std::move(InnerJoin(std::forward(leftField), std::forward(rightField), cond, std::move(qr))); } - Query &InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) & { - return Join(JoinType::InnerJoin, index, joinIndex, cond, OpAnd, qr); + template + Query &InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) & { + return Join(JoinType::InnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, qr); } - Query &&InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) && { - return std::move(InnerJoin(index, joinIndex, cond, qr)); + template + Query &&InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) && { + return std::move(InnerJoin(std::forward(leftField), std::forward(rightField), cond, qr)); } /// Left Join of this namespace with another one. - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param qr - query of the namespace that is going to be joined with this one. /// @return Query object ready to be executed. - Query &LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) & { - return Join(JoinType::LeftJoin, index, joinIndex, cond, OpAnd, std::move(qr)); + template + Query &LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) & { + return Join(JoinType::LeftJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, std::move(qr)); } - Query &&LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) && { - return std::move(LeftJoin(index, joinIndex, cond, std::move(qr))); + template + Query &&LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) && { + return std::move(LeftJoin(std::forward(leftField), std::forward(rightField), cond, std::move(qr))); } - Query &LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) & { - return Join(JoinType::LeftJoin, index, joinIndex, cond, OpAnd, qr); + template + Query &LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) & { + return Join(JoinType::LeftJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, qr); } - Query &&LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) && { - return std::move(LeftJoin(index, joinIndex, cond, qr)); + template + Query &&LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) && { + return std::move(LeftJoin(std::forward(leftField), std::forward(rightField), cond, qr)); } /// OrInnerJoin of this namespace with another one. - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param qr - query of the namespace that is going to be joined with this one. /// @return a reference to a query object ready to be executed. - Query &OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) & { - return Join(JoinType::OrInnerJoin, index, joinIndex, cond, OpAnd, std::move(qr)); + template + Query &OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) & { + return Join(JoinType::OrInnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, std::move(qr)); } - Query &&OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) && { - return std::move(OrInnerJoin(index, joinIndex, cond, std::move(qr))); + template + Query &&OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) && { + return std::move(OrInnerJoin(std::forward(leftField), std::forward(rightField), cond, std::move(qr))); } - Query &OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) & { - return Join(JoinType::OrInnerJoin, index, joinIndex, cond, OpAnd, qr); + template + Query &OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) & { + return Join(JoinType::OrInnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, qr); } - Query &&OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) && { - return std::move(OrInnerJoin(index, joinIndex, cond, qr)); + template + Query &&OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) && { + return std::move(OrInnerJoin(std::forward(leftField), std::forward(rightField), cond, qr)); } Query &Merge(const Query &q) &; Query &&Merge(const Query &q) && { return std::move(Merge(q)); } @@ -491,11 +528,15 @@ class Query { /// @param sort - sorting column name. /// @param desc - is sorting direction descending or ascending. /// @return Query object. - Query &Sort(std::string sort, bool desc) & { // -V1071 - if (sort.length()) sortingEntries_.emplace_back(std::move(sort), desc); + template + Query &Sort(Str &&sort, bool desc) & { // -V1071 + if (!strEmpty(sort)) sortingEntries_.emplace_back(std::forward(sort), desc); return *this; } - Query &&Sort(std::string sort, bool desc) && { return std::move(Sort(std::move(sort), desc)); } + template + Query &&Sort(Str &&sort, bool desc) && { + return std::move(Sort(std::forward(sort), desc)); + } /// Performs sorting by ST_Distance() expressions for geometry index. Sorting function will use distance between field and target point. /// @param field - field's name. This field must contain Point. @@ -519,17 +560,17 @@ class Query { /// @param desc - is sorting direction descending or ascending. /// @param forcedSortOrder - list of values for forced sort order. /// @return Query object. - template - Query &Sort(std::string sort, bool desc, std::initializer_list forcedSortOrder) & { + template + Query &Sort(Str &&sort, bool desc, std::initializer_list forcedSortOrder) & { if (!sortingEntries_.empty() && !std::empty(forcedSortOrder)) throw Error(errParams, "Forced sort order is allowed for the first sorting entry only"); - sortingEntries_.emplace_back(std::move(sort), desc); + sortingEntries_.emplace_back(std::forward(sort), desc); for (const T &v : forcedSortOrder) forcedSortOrder_.emplace_back(v); return *this; } - template - Query &&Sort(std::string sort, bool desc, std::initializer_list forcedSortOrder) && { - return std::move(Sort(std::move(sort), desc, std::move(forcedSortOrder))); + template + Query &&Sort(Str &&sort, bool desc, std::initializer_list forcedSortOrder) && { + return std::move(Sort(std::forward(sort), desc, std::move(forcedSortOrder))); } /// Performs sorting by certain column. Analog to sql ORDER BY. @@ -537,28 +578,32 @@ class Query { /// @param desc - is sorting direction descending or ascending. /// @param forcedSortOrder - list of values for forced sort order. /// @return Query object. - template - Query &Sort(std::string sort, bool desc, const T &forcedSortOrder) & { + template + Query &Sort(Str &&sort, bool desc, const T &forcedSortOrder) & { if (!sortingEntries_.empty() && !forcedSortOrder.empty()) throw Error(errParams, "Forced sort order is allowed for the first sorting entry only"); - sortingEntries_.emplace_back(std::move(sort), desc); + sortingEntries_.emplace_back(std::forward(sort), desc); for (const auto &v : forcedSortOrder) forcedSortOrder_.emplace_back(v); return *this; } - template - Query &&Sort(std::string sort, bool desc, const T &forcedSortOrder) && { - return std::move(Sort(std::move(sort), desc, forcedSortOrder)); + template + Query &&Sort(Str &&sort, bool desc, const T &forcedSortOrder) && { + return std::move(Sort(std::forward(sort), desc, forcedSortOrder)); } /// Performs distinct for a certain index. /// @param indexName - name of index for distict operation. - Query &Distinct(std::string indexName) & { - if (indexName.length()) { - aggregations_.emplace_back(AggDistinct, h_vector{std::move(indexName)}); + template + Query &Distinct(Str &&indexName) & { + if (!strEmpty(indexName)) { + aggregations_.emplace_back(AggDistinct, h_vector{std::forward(indexName)}); } return *this; } - Query &&Distinct(std::string indexName) && { return std::move(Distinct(std::move(indexName))); } + template + Query &&Distinct(Str &&indexName) && { + return std::move(Distinct(std::forward(indexName))); + } /// Sets list of columns in this namespace to be finally selected. /// @param l - list of columns to be selected. @@ -636,7 +681,7 @@ class Query { /// @param limit - number of rows to get from result set. /// @return Query object. Query &Limit(unsigned limit) &noexcept { - count = limit; + count_ = limit; return *this; } Query &&Limit(unsigned limit) &&noexcept { return std::move(Limit(limit)); } @@ -646,7 +691,7 @@ class Query { /// @param offset - index of the first row to get from result set. /// @return Query object. Query &Offset(unsigned offset) &noexcept { - start = offset; + start_ = offset; return *this; } Query &&Offset(unsigned offset) &&noexcept { return std::move(Offset(offset)); } @@ -654,7 +699,7 @@ class Query { /// Set the total count calculation mode to Accurate /// @return Query object Query &ReqTotal() &noexcept { - calcTotal = ModeAccurateTotal; + calcTotal_ = ModeAccurateTotal; return *this; } Query &&ReqTotal() &&noexcept { return std::move(ReqTotal()); } @@ -663,7 +708,7 @@ class Query { /// It will be use LRUCache for total count result /// @return Query object Query &CachedTotal() &noexcept { - calcTotal = ModeCachedTotal; + calcTotal_ = ModeCachedTotal; return *this; } Query &&CachedTotal() &&noexcept { return std::move(CachedTotal()); } @@ -696,38 +741,47 @@ class Query { void WalkNested(bool withSelf, bool withMerged, const std::function &visitor) const; - bool HasLimit() const noexcept { return count != QueryEntry::kDefaultLimit; } - bool HasOffset() const noexcept { return start != QueryEntry::kDefaultOffset; } + bool HasLimit() const noexcept { return count_ != QueryEntry::kDefaultLimit; } + bool HasOffset() const noexcept { return start_ != QueryEntry::kDefaultOffset; } bool IsWALQuery() const noexcept; const std::vector &UpdateFields() const noexcept { return updateFields_; } QueryType Type() const noexcept { return type_; } - -protected: - void deserialize(Serializer &ser, bool &hasJoinConditions); - -public: - std::string _namespace; /// Name of the namespace. - unsigned start = QueryEntry::kDefaultOffset; /// First row index from result set. - unsigned count = QueryEntry::kDefaultLimit; /// Number of rows from result set. - int debugLevel = 0; /// Debug level. - StrictMode strictMode = StrictModeNotSet; /// Strict mode. - bool explain_ = false; /// Explain query if true - CalcTotalMode calcTotal = ModeNoTotal; /// Calculation mode. - QueryType type_ = QuerySelect; /// Query type - OpType nextOp_ = OpAnd; /// Next operation constant. - SortingEntries sortingEntries_; /// Sorting data. - std::vector forcedSortOrder_; /// Keys that always go first - before any ordered values. - std::vector joinQueries_; /// List of queries for join. - std::vector mergeQueries_; /// List of merge queries. - h_vector selectFilter_; /// List of columns in a final result set. - std::vector selectFunctions_; /// List of sql functions + const std::string &NsName() const &noexcept { return namespace_; } + template + void SetNsName(T &&nsName) &noexcept { + namespace_ = std::forward(nsName); + } + unsigned Limit() const noexcept { return count_; } + unsigned Offset() const noexcept { return start_; } + CalcTotalMode CalcTotal() const noexcept { return calcTotal_; } + void CalcTotal(CalcTotalMode calcTotal) noexcept { calcTotal_ = calcTotal; } + + int debugLevel = 0; /// Debug level. + StrictMode strictMode = StrictModeNotSet; /// Strict mode. + bool explain_ = false; /// Explain query if true + QueryType type_ = QuerySelect; /// Query type + OpType nextOp_ = OpAnd; /// Next operation constant. + SortingEntries sortingEntries_; /// Sorting data. + std::vector forcedSortOrder_; /// Keys that always go first - before any ordered values. + std::vector joinQueries_; /// List of queries for join. + std::vector mergeQueries_; /// List of merge queries. + h_vector selectFilter_; /// List of columns in a final result set. + std::vector selectFunctions_; /// List of sql functions QueryEntries entries; std::vector aggregations_; + auto NsName() const && = delete; + private: - std::vector updateFields_; /// List of fields (and values) for update. + void deserialize(Serializer &ser, bool &hasJoinConditions); + + std::string namespace_; /// Name of the namespace. + unsigned start_ = QueryEntry::kDefaultOffset; /// First row index from result set. + unsigned count_ = QueryEntry::kDefaultLimit; /// Number of rows from result set. + CalcTotalMode calcTotal_ = ModeNoTotal; /// Calculation mode. + std::vector updateFields_; /// List of fields (and values) for update. bool withRank_ = false; friend class SQLParser; }; diff --git a/cpp_src/core/query/queryentry.cc b/cpp_src/core/query/queryentry.cc index a7cc7f4e3..d7148dc49 100644 --- a/cpp_src/core/query/queryentry.cc +++ b/cpp_src/core/query/queryentry.cc @@ -8,7 +8,6 @@ #include "query.h" #include "tools/serializer.h" #include "tools/string_regexp_functions.h" -#include "tools/stringstools.h" namespace reindexer { @@ -21,11 +20,11 @@ std::string JoinQueryEntry::Dump(const std::vector &joinedSelectors) const { ser << '('; for (const auto &jqe : q.joinEntries_) { if (&jqe != &q.joinEntries_.front()) { - ser << ' ' << jqe.op_ << ' '; + ser << ' ' << jqe.Operation() << ' '; } else { - assertrx(jqe.op_ == OpAnd); + assertrx(jqe.Operation() == OpAnd); } - ser << q._namespace << '.' << jqe.joinIndex_ << ' ' << InvertJoinCondition(jqe.condition_) << ' ' << jqe.index_; + ser << q.NsName() << '.' << jqe.RightFieldName() << ' ' << InvertJoinCondition(jqe.Condition()) << ' ' << jqe.LeftFieldName(); } ser << ')'; return std::string{ser.Slice()}; @@ -41,30 +40,133 @@ std::string JoinQueryEntry::DumpOnCondition(const std::vector &joinedSelecto ser << js.Type() << " ON ("; for (const auto &jqe : q.joinEntries_) { if (&jqe != &q.joinEntries_.front()) { - ser << ' ' << jqe.op_ << ' '; + ser << ' ' << jqe.Operation() << ' '; } - ser << q._namespace << '.' << jqe.joinIndex_ << ' ' << InvertJoinCondition(jqe.condition_) << ' ' << jqe.index_; + ser << q.NsName() << '.' << jqe.RightFieldName() << ' ' << InvertJoinCondition(jqe.Condition()) << ' ' << jqe.LeftFieldName(); } ser << ')'; return std::string{ser.Slice()}; } template std::string JoinQueryEntry::DumpOnCondition(const JoinedSelectors &) const; -bool QueryEntry::operator==(const QueryEntry &obj) const { - return condition == obj.condition && index == obj.index && idxNo == obj.idxNo && distinct == obj.distinct && - values.RelaxCompare(obj.values) == 0; +bool QueryField::operator==(const QueryField &other) const noexcept { + if (fieldName_ != other.fieldName_ || idxNo_ != other.idxNo_ || fieldsSet_ != other.fieldsSet_ || + !fieldType_.IsSame(other.fieldType_) || !selectType_.IsSame(other.selectType_) || + compositeFieldsTypes_.size() != other.compositeFieldsTypes_.size()) { + return false; + } + for (size_t i = 0, s = compositeFieldsTypes_.size(); i < s; ++i) { + if (!compositeFieldsTypes_[i].IsSame(other.compositeFieldsTypes_[i])) { + return false; + } + } + return true; +} + +void QueryField::SetField(FieldsSet &&fields) & { + assertrx_throw(fields.size() == 1); + assertrx_throw(fields[0] == IndexValueType::SetByJsonPath); + assertrx_throw(idxNo_ == IndexValueType::NotSet); + idxNo_ = IndexValueType::SetByJsonPath; + fieldsSet_ = std::move(fields); +} + +static void checkIndexData([[maybe_unused]] int idxNo, [[maybe_unused]] const FieldsSet &fields, KeyValueType fieldType, + [[maybe_unused]] const std::vector &compositeFieldsTypes) { + assertrx_throw(idxNo >= 0); + if (fieldType.Is()) { + assertrx_throw(fields.size() == compositeFieldsTypes.size()); + } else { + assertrx_throw(fields.size() == 1); + assertrx_throw(compositeFieldsTypes.empty()); + } +} + +void QueryField::SetIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) & { + checkIndexData(idxNo, fields, fieldType, compositeFieldsTypes); + idxNo_ = idxNo; + fieldsSet_ = std::move(fields); + fieldType_ = fieldType; + selectType_ = selectType; + compositeFieldsTypes_ = std::move(compositeFieldsTypes); +} + +bool QueryField::HaveEmptyField() const noexcept { + size_t tagsNo = 0; + for (auto f : Fields()) { + if (f == IndexValueType::SetByJsonPath) { + if (Fields().getTagsPath(tagsNo).empty()) { + return true; + } + ++tagsNo; + } + } + return Fields().empty(); +} + +bool QueryEntry::operator==(const QueryEntry &other) const noexcept { + return QueryField::operator==(other) && condition_ == other.condition_ && distinct_ == other.distinct_ && + values_.RelaxCompare(other.values_) == 0; } +template +void QueryEntry::verify(CondType cond, const VariantArray &values) { + if constexpr (flags & kIgnoreEmptyValues) { + if (values.empty()) { + return; + } + } + const auto checkArgsCount = [&](size_t argsCountReq) { + if (values.size() != argsCountReq) { + throw Error{errLogic, "Condition %s must have exact %d argument, but %d arguments was provided", CondTypeToStr(cond), + argsCountReq, values.size()}; + } + }; + switch (cond) { + case CondEq: + case CondSet: + case CondAllSet: + break; + case CondAny: + case CondEmpty: + if (!values.empty() && !(values.size() == 1 && values[0].Type().Is())) { + throw Error{errLogic, "Condition %s must have no argument or single null argument, but %d not null arguments was provided", + CondTypeToStr(cond), values.size()}; + } + break; + case CondGe: + case CondGt: + case CondLt: + case CondLe: + checkArgsCount(1); + break; + case CondLike: + checkArgsCount(1); + if (!values[0].Type().Is()) { + throw Error{errLogic, "Condition %s must have string argument, but %d argument was provided", CondTypeToStr(cond), + values[0].Type().Name()}; + } + break; + case CondRange: + case CondDWithin: + checkArgsCount(2); + break; + } +} +template void QueryEntry::verify<0u>(CondType, const VariantArray &); +template void QueryEntry::verify(CondType, const VariantArray &); + std::string QueryEntry::Dump() const { WrSerializer ser; - if (distinct) { - ser << "Distinct index: " << index; + if (Distinct()) { + ser << "Distinct index: " << FieldName(); } else { - ser << index << ' ' << condition << ' '; - const bool severalValues = (values.size() > 1); + ser << FieldName() << ' ' << condition_ << ' '; + const bool severalValues = (Values().size() > 1); if (severalValues) ser << '('; - for (auto &v : values) { - if (&v != &*values.begin()) ser << ','; + for (auto &v : Values()) { + if (&v != &*Values().begin()) ser << ','; ser << '\'' << v.As() << '\''; } if (severalValues) ser << ')'; @@ -75,18 +177,18 @@ std::string QueryEntry::Dump() const { std::string QueryEntry::DumpBrief() const { WrSerializer ser; { - ser << index << ' ' << condition << ' '; - const bool severalValues = (values.size() > 1); + ser << FieldName() << ' ' << Condition() << ' '; + const bool severalValues = (Values().size() > 1); if (severalValues) { ser << "(...)"; } else { - ser << '\'' << values.front().As() << '\''; + ser << '\'' << Values().front().As() << '\''; } } return std::string(ser.Slice()); } -AggregateEntry::AggregateEntry(AggType type, h_vector fields, SortingEntries sort, unsigned limit, unsigned offset) +AggregateEntry::AggregateEntry(AggType type, h_vector &&fields, SortingEntries &&sort, unsigned limit, unsigned offset) : type_(type), fields_(std::move(fields)), sortingEntries_{std::move(sort)}, limit_(limit), offset_(offset) { switch (type_) { case AggFacet: @@ -134,7 +236,7 @@ AggregateEntry::AggregateEntry(AggType type, h_vector fields, So } } -void AggregateEntry::AddSortingEntry(SortingEntry sorting) { +void AggregateEntry::AddSortingEntry(SortingEntry &&sorting) { if (type_ != AggFacet) { throw Error(errQueryExec, "Sort is not available for aggregation %s", AggTypeToStr(type_)); } @@ -155,21 +257,13 @@ void AggregateEntry::SetOffset(unsigned o) { offset_ = o; } -BetweenFieldsQueryEntry::BetweenFieldsQueryEntry(std::string fstIdx, CondType cond, std::string sndIdx) - : firstIndex{std::move(fstIdx)}, secondIndex{std::move(sndIdx)}, condition_{cond} { - if (condition_ == CondAny || condition_ == CondEmpty || condition_ == CondDWithin) { - throw Error{errLogic, "Condition '%s' is inapplicable between two fields", std::string{CondTypeToStr(condition_)}}; - } -} - bool BetweenFieldsQueryEntry::operator==(const BetweenFieldsQueryEntry &other) const noexcept { - return firstIdxNo == other.firstIdxNo && secondIdxNo == other.secondIdxNo && Condition() == other.Condition() && - firstIndex == other.firstIndex && secondIndex == other.secondIndex; + return leftField_ == other.leftField_ && rightField_ == other.rightField_ && Condition() == other.Condition(); } std::string BetweenFieldsQueryEntry::Dump() const { WrSerializer ser; - ser << firstIndex << ' ' << Condition() << ' ' << secondIndex; + ser << LeftFieldName() << ' ' << Condition() << ' ' << RightFieldName(); return std::string{ser.Slice()}; } @@ -184,30 +278,31 @@ void QueryEntries::serialize(const_iterator it, const_iterator to, WrSerializer ser.PutVarUint(QueryCloseBracket); }, [&ser, op](const QueryEntry &entry) { - entry.distinct ? ser.PutVarUint(QueryDistinct) : ser.PutVarUint(QueryCondition); - ser.PutVString(entry.index); - if (entry.distinct) return; + entry.Distinct() ? ser.PutVarUint(QueryDistinct) : ser.PutVarUint(QueryCondition); + ser.PutVString(entry.FieldName()); + if (entry.Distinct()) return; ser.PutVarUint(op); - ser.PutVarUint(entry.condition); - if (entry.condition == CondDWithin) { - if (entry.values.size() != 2) { - throw Error(errLogic, "Condition DWithin must have exact 2 value, but %d values was provided", entry.values.size()); + ser.PutVarUint(entry.Condition()); + if (entry.Condition() == CondDWithin) { + if (entry.Values().size() != 2) { + throw Error(errLogic, "Condition DWithin must have exact 2 value, but %d values was provided", + entry.Values().size()); } ser.PutVarUint(3); - if (entry.values[0].Type().Is()) { - const Point point = static_cast(entry.values[0]); + if (entry.Values()[0].Type().Is()) { + const Point point = static_cast(entry.Values()[0]); ser.PutDouble(point.X()); ser.PutDouble(point.Y()); - ser.PutVariant(entry.values[1]); + ser.PutVariant(entry.Values()[1]); } else { - const Point point = static_cast(entry.values[1]); + const Point point = static_cast(entry.Values()[1]); ser.PutDouble(point.X()); ser.PutDouble(point.Y()); - ser.PutVariant(entry.values[0]); + ser.PutVariant(entry.Values()[0]); } } else { - ser.PutVarUint(entry.values.size()); - for (auto &kv : entry.values) ser.PutVariant(kv); + ser.PutVarUint(entry.Values().size()); + for (auto &kv : entry.Values()) ser.PutVariant(kv); } }, [&ser, op](const JoinQueryEntry &jqe) { @@ -218,9 +313,9 @@ void QueryEntries::serialize(const_iterator it, const_iterator to, WrSerializer [&ser, op](const BetweenFieldsQueryEntry &entry) { ser.PutVarUint(QueryBetweenFieldsCondition); ser.PutVarUint(op); - ser.PutVString(entry.firstIndex); + ser.PutVString(entry.LeftFieldName()); ser.PutVarUint(entry.Condition()); - ser.PutVString(entry.secondIndex); + ser.PutVString(entry.RightFieldName()); }, [&ser, op](const AlwaysFalse &) { ser.PutVarUint(QueryAlwaysFalseCondition); @@ -233,13 +328,8 @@ bool UpdateEntry::operator==(const UpdateEntry &obj) const noexcept { return isExpression_ == obj.isExpression_ && column_ == obj.column_ && mode_ == obj.mode_ && values_ == obj.values_; } -bool QueryJoinEntry::operator==(const QueryJoinEntry &obj) const noexcept { - if (op_ != obj.op_) return false; - if (static_cast(condition_) != obj.condition_) return false; - if (index_ != obj.index_) return false; - if (joinIndex_ != obj.joinIndex_) return false; - if (idxNo != obj.idxNo) return false; - return true; +bool QueryJoinEntry::operator==(const QueryJoinEntry &other) const noexcept { + return op_ == other.op_ && condition_ == other.condition_ && leftField_ == other.leftField_ && rightField_ == other.rightField_; } bool AggregateEntry::operator==(const AggregateEntry &obj) const noexcept { @@ -254,7 +344,7 @@ bool SortingEntry::operator==(const SortingEntry &obj) const noexcept { return true; } -bool QueryEntries::checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &pl, TagsMatcher &tagsMatcher) { +bool QueryEntries::checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &pl) { assertrx(begin != end && begin->operation != OpOr); bool result = true; for (auto it = begin; it != end; ++it) { @@ -264,40 +354,26 @@ bool QueryEntries::checkIfSatisfyConditions(const_iterator begin, const_iterator break; } const bool lastResult = it->InvokeAppropriate( - [&it, &pl, &tagsMatcher](const QueryEntriesBracket &) { - return checkIfSatisfyConditions(it.cbegin(), it.cend(), pl, tagsMatcher); - }, - [&pl, &tagsMatcher](const QueryEntry &qe) { return checkIfSatisfyCondition(qe, pl, tagsMatcher); }, - [&pl, &tagsMatcher](const BetweenFieldsQueryEntry &qe) { return checkIfSatisfyCondition(qe, pl, tagsMatcher); }, + [&it, &pl](const QueryEntriesBracket &) { return checkIfSatisfyConditions(it.cbegin(), it.cend(), pl); }, + [&pl](const QueryEntry &qe) { return checkIfSatisfyCondition(qe, pl); }, + [&pl](const BetweenFieldsQueryEntry &qe) { return checkIfSatisfyCondition(qe, pl); }, [](const JoinQueryEntry &) -> bool { abort(); }, [](const AlwaysFalse &) { return false; }); result = (lastResult != (it->operation == OpNot)); } return result; } -bool QueryEntries::checkIfSatisfyCondition(const QueryEntry &qEntry, const ConstPayload &pl, TagsMatcher &tagsMatcher) { +bool QueryEntries::checkIfSatisfyCondition(const QueryEntry &qEntry, const ConstPayload &pl) { VariantArray values; - if (qEntry.idxNo == IndexValueType::SetByJsonPath) { - pl.GetByJsonPath(qEntry.index, tagsMatcher, values, KeyValueType::Undefined{}); - } else { - pl.Get(qEntry.idxNo, values); - } - return checkIfSatisfyCondition(values, qEntry.condition, qEntry.values); + pl.GetByFieldsSet(qEntry.Fields(), values, qEntry.FieldType(), qEntry.CompositeFieldsTypes()); + return checkIfSatisfyCondition(values, qEntry.Condition(), qEntry.Values()); } -bool QueryEntries::checkIfSatisfyCondition(const BetweenFieldsQueryEntry &qEntry, const ConstPayload &pl, TagsMatcher &tagsMatcher) { +bool QueryEntries::checkIfSatisfyCondition(const BetweenFieldsQueryEntry &qEntry, const ConstPayload &pl) { VariantArray lValues; - if (qEntry.firstIdxNo == IndexValueType::SetByJsonPath) { - pl.GetByJsonPath(qEntry.firstIndex, tagsMatcher, lValues, KeyValueType::Undefined{}); - } else { - pl.Get(qEntry.firstIdxNo, lValues); - } + pl.GetByFieldsSet(qEntry.LeftFields(), lValues, qEntry.LeftFieldType(), qEntry.LeftCompositeFieldsTypes()); VariantArray rValues; - if (qEntry.secondIdxNo == IndexValueType::SetByJsonPath) { - pl.GetByJsonPath(qEntry.secondIndex, tagsMatcher, rValues, KeyValueType::Undefined{}); - } else { - pl.Get(qEntry.secondIdxNo, rValues); - } + pl.GetByFieldsSet(qEntry.RightFields(), rValues, qEntry.RightFieldType(), qEntry.RightCompositeFieldsTypes()); return checkIfSatisfyCondition(lValues, qEntry.Condition(), rValues); } @@ -350,18 +426,11 @@ bool QueryEntries::checkIfSatisfyCondition(const VariantArray &lValues, CondType return false; } case CondType::CondRange: - if (rValues.size() != 2) throw Error(errParams, "For ranged query reuqired 2 arguments, but provided %d", rValues.size()); for (const auto &v : lValues) { if (v.RelaxCompare(rValues[0]) < 0 || v.RelaxCompare(rValues[1]) > 0) return false; } return true; case CondType::CondLike: - if (rValues.size() != 1) { - throw Error(errLogic, "Condition LIKE must have exact 1 value, but %d values was provided", rValues.size()); - } - if (!rValues[0].Type().Is()) { - throw Error(errLogic, "Condition LIKE must have value of string type, but %s value was provided", rValues[0].Type().Name()); - } for (const auto &v : lValues) { if (!v.Type().Is()) { throw Error(errLogic, "Condition LIKE must be applied to data of string type, but %s was provided", v.Type().Name()); @@ -370,9 +439,6 @@ bool QueryEntries::checkIfSatisfyCondition(const VariantArray &lValues, CondType } return false; case CondType::CondDWithin: { - if (rValues.size() != 2) { - throw Error(errLogic, "Condition DWithin must have exact 2 value, but %d values was provided", rValues.size()); - } Point point; double distance; if (rValues[0].Type().Is()) { @@ -390,4 +456,56 @@ bool QueryEntries::checkIfSatisfyCondition(const VariantArray &lValues, CondType return false; } +template +std::string QueryJoinEntry::DumpCondition(const JS &joinedSelector, bool needOp) const { + WrSerializer ser; + const auto &q = joinedSelector.JoinQuery(); + if (needOp) { + ser << ' ' << op_ << ' '; + } + ser << q.NsName() << '.' << RightFieldName() << ' ' << InvertJoinCondition(condition_) << ' ' << LeftFieldName(); + return std::string{ser.Slice()}; +} +template std::string QueryJoinEntry::DumpCondition(const JoinedSelector &, bool) const; + +void QueryEntries::dumpEqualPositions(size_t level, WrSerializer &ser, const EqualPositions_t &equalPositions) { + for (const auto &eq : equalPositions) { + for (size_t i = 0; i < level; ++i) { + ser << " "; + } + ser << "equal_poisition("; + for (size_t i = 0, s = eq.size(); i < s; ++i) { + if (i != 0) ser << ", "; + ser << eq[i]; + } + ser << ")\n"; + } +} + +template +void QueryEntries::dump(size_t level, const_iterator begin, const_iterator end, const std::vector &joinedSelectors, WrSerializer &ser) { + for (const_iterator it = begin; it != end; ++it) { + for (size_t i = 0; i < level; ++i) { + ser << " "; + } + if (it != begin || it->operation != OpAnd) { + ser << it->operation << ' '; + } + it->InvokeAppropriate( + [&](const QueryEntriesBracket &b) { + ser << "(\n"; + dump(level + 1, it.cbegin(), it.cend(), joinedSelectors, ser); + dumpEqualPositions(level + 1, ser, b.equalPositions); + for (size_t i = 0; i < level; ++i) { + ser << " "; + } + ser << ")\n"; + }, + [&ser](const QueryEntry &qe) { ser << qe.Dump() << '\n'; }, + [&joinedSelectors, &ser](const JoinQueryEntry &jqe) { ser << jqe.Dump(joinedSelectors) << '\n'; }, + [&ser](const BetweenFieldsQueryEntry &qe) { ser << qe.Dump() << '\n'; }, + [&ser](const AlwaysFalse &) { ser << "AlwaysFalse" << 'n'; }); + } +} + } // namespace reindexer diff --git a/cpp_src/core/query/queryentry.h b/cpp_src/core/query/queryentry.h index d2b53ebf8..6d00faf35 100644 --- a/cpp_src/core/query/queryentry.h +++ b/cpp_src/core/query/queryentry.h @@ -5,10 +5,11 @@ #include #include "core/expressiontree.h" #include "core/keyvalue/variant.h" +#include "core/payload/fieldsset.h" #include "core/type_consts.h" -#include "core/type_consts_helpers.h" #include "estl/h_vector.h" #include "tools/serializer.h" +#include "tools/verifying_updater.h" namespace reindexer { @@ -31,44 +32,181 @@ struct JoinQueryEntry { std::string DumpOnCondition(const std::vector &joinedSelectors) const; }; -struct QueryEntry { +class QueryField { +public: + template + explicit QueryField(Str &&fieldName) noexcept : fieldName_{std::forward(fieldName)} {} + QueryField(std::string &&fieldName, int idxNo, FieldsSet fields, KeyValueType fieldType, + std::vector &&compositeFieldsTypes); + QueryField(QueryField &&) noexcept = default; + QueryField(const QueryField &) = default; + QueryField &operator=(QueryField &&) noexcept = default; + + [[nodiscard]] bool operator==(const QueryField &) const noexcept; + [[nodiscard]] bool operator!=(const QueryField &other) const noexcept { return !operator==(other); } + + [[nodiscard]] int IndexNo() const noexcept { return idxNo_; } + [[nodiscard]] bool IsFieldIndexed() const noexcept { return idxNo_ >= 0; } + [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return idxNo_ != IndexValueType::NotSet; } + [[nodiscard]] const FieldsSet &Fields() const &noexcept { return fieldsSet_; } + [[nodiscard]] const std::string &FieldName() const &noexcept { return fieldName_; } + [[nodiscard]] KeyValueType FieldType() const noexcept { return fieldType_; } + [[nodiscard]] KeyValueType SelectType() const noexcept { return selectType_; } + [[nodiscard]] const std::vector &CompositeFieldsTypes() const &noexcept { return compositeFieldsTypes_; } + [[nodiscard]] bool HaveEmptyField() const noexcept; + void SetField(FieldsSet &&fields) &; + void SetIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) &; + + QueryField &operator=(const QueryField &) = delete; + auto Fields() const && = delete; + auto FieldName() const && = delete; + auto CompositeFieldsTypes() const && = delete; + +private: + std::string fieldName_; + int idxNo_{IndexValueType::NotSet}; + FieldsSet fieldsSet_; + KeyValueType fieldType_{KeyValueType::Undefined{}}; + KeyValueType selectType_{KeyValueType::Undefined{}}; + std::vector compositeFieldsTypes_; +}; + +class QueryEntry : private QueryField { + static constexpr unsigned kIgnoreEmptyValues = 1u; + +public: + struct DistinctTag {}; + struct IgnoreEmptyValues {}; static constexpr unsigned kDefaultLimit = UINT_MAX; static constexpr unsigned kDefaultOffset = 0; - QueryEntry(std::string idx, CondType cond, VariantArray v) : index{std::move(idx)}, condition{cond}, values(std::move(v)) {} - QueryEntry(CondType cond, std::string idx, int idxN, bool dist = false) - : index(std::move(idx)), idxNo(idxN), condition(cond), distinct(dist) {} - QueryEntry() = default; + template + QueryEntry(Str &&fieldName, CondType cond, VariantArray &&v) + : QueryField{std::forward(fieldName)}, values_{std::move(v)}, condition_{cond} { + Verify(); + } + template + QueryEntry(Str &&fieldName, DistinctTag) : QueryField{std::forward(fieldName)}, condition_{CondAny}, distinct_{true} { + Verify(); + } + QueryEntry(QueryField &&field, CondType cond, VariantArray &&v) + : QueryField{std::move(field)}, values_{std::move(v)}, condition_{cond} { + Verify(); + } + QueryEntry(QueryField &&field, CondType cond, IgnoreEmptyValues) : QueryField{std::move(field)}, condition_{cond} { + verifyIgnoringEmptyValues(); + } + [[nodiscard]] CondType Condition() const noexcept { return condition_; } + [[nodiscard]] const VariantArray &Values() const &noexcept { return values_; } + [[nodiscard]] VariantArray &&Values() &&noexcept { return std::move(values_); } + [[nodiscard]] auto UpdatableValues(IgnoreEmptyValues) &noexcept { + return VerifyingUpdater{*this}; + } + [[nodiscard]] bool Distinct() const noexcept { return distinct_; } + void Distinct(bool d) noexcept { distinct_ = d; } + using QueryField::IndexNo; + using QueryField::IsFieldIndexed; + using QueryField::FieldsHaveBeenSet; + using QueryField::Fields; + using QueryField::FieldName; + using QueryField::FieldType; + using QueryField::SelectType; + using QueryField::CompositeFieldsTypes; + using QueryField::SetField; + using QueryField::SetIndexData; + using QueryField::HaveEmptyField; + void SetCondAndValues(CondType cond, VariantArray &&values) { + verify(cond, values); + condition_ = cond; + values_ = std::move(values); + } - bool operator==(const QueryEntry &) const; - bool operator!=(const QueryEntry &other) const { return !operator==(other); } + const QueryField &FieldData() const &noexcept { return static_cast(*this); } + QueryField &FieldData() &noexcept { return static_cast(*this); } + void ConvertValuesToFieldType() & { + for (Variant &v : values_) { + v.convert(SelectType()); + } + } + void ConvertValuesToFieldType(const PayloadType &pt) & { + if (SelectType().Is() || Condition() == CondDWithin) { + return; + } + for (Variant &v : values_) { + v.convert(SelectType(), &pt, &Fields()); + } + } + void Verify() const { verify(condition_, values_); } - std::string index; - int idxNo = IndexValueType::NotSet; - CondType condition = CondType::CondAny; - bool distinct = false; - VariantArray values; + [[nodiscard]] bool operator==(const QueryEntry &) const noexcept; + [[nodiscard]] bool operator!=(const QueryEntry &other) const noexcept { return !operator==(other); } - std::string Dump() const; - std::string DumpBrief() const; + [[nodiscard]] std::string Dump() const; + [[nodiscard]] std::string DumpBrief() const; + + auto Values() const && = delete; + auto FieldData() const && = delete; + +private: + template + static void verify(CondType, const VariantArray &); + void verifyIgnoringEmptyValues() const { verify(condition_, values_); } + + VariantArray values_; + CondType condition_; + bool distinct_{false}; }; +extern template void QueryEntry::verify<0u>(CondType, const VariantArray &); +extern template void QueryEntry::verify(CondType, const VariantArray &); class BetweenFieldsQueryEntry { public: - BetweenFieldsQueryEntry(std::string fstIdx, CondType cond, std::string sndIdx); - - bool operator==(const BetweenFieldsQueryEntry &) const noexcept; - bool operator!=(const BetweenFieldsQueryEntry &other) const noexcept { return !operator==(other); } - - std::string firstIndex; - std::string secondIndex; - int firstIdxNo = IndexValueType::NotSet; - int secondIdxNo = IndexValueType::NotSet; + template + BetweenFieldsQueryEntry(StrL &&fstIdx, CondType cond, StrR &&sndIdx) + : leftField_{std::forward(fstIdx)}, rightField_{std::forward(sndIdx)}, condition_{cond} { + if (condition_ == CondAny || condition_ == CondEmpty || condition_ == CondDWithin) { + throw Error{errLogic, "Condition '%s' is inapplicable between two fields", std::string{CondTypeToStr(condition_)}}; + } + } - CondType Condition() const noexcept { return condition_; } - std::string Dump() const; + [[nodiscard]] bool operator==(const BetweenFieldsQueryEntry &) const noexcept; + [[nodiscard]] bool operator!=(const BetweenFieldsQueryEntry &other) const noexcept { return !operator==(other); } + + [[nodiscard]] CondType Condition() const noexcept { return condition_; } + [[nodiscard]] int LeftIdxNo() const noexcept { return leftField_.IndexNo(); } + [[nodiscard]] int RightIdxNo() const noexcept { return rightField_.IndexNo(); } + [[nodiscard]] const std::string &LeftFieldName() const &noexcept { return leftField_.FieldName(); } + [[nodiscard]] const std::string &RightFieldName() const &noexcept { return rightField_.FieldName(); } + [[nodiscard]] const FieldsSet &LeftFields() const &noexcept { return leftField_.Fields(); } + [[nodiscard]] const FieldsSet &RightFields() const &noexcept { return rightField_.Fields(); } + [[nodiscard]] KeyValueType LeftFieldType() const noexcept { return leftField_.FieldType(); } + [[nodiscard]] KeyValueType RightFieldType() const noexcept { return rightField_.FieldType(); } + [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const &noexcept { return leftField_.CompositeFieldsTypes(); } + [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const &noexcept { + return rightField_.CompositeFieldsTypes(); + } + [[nodiscard]] const QueryField &LeftFieldData() const &noexcept { return leftField_; } + [[nodiscard]] QueryField &LeftFieldData() &noexcept { return leftField_; } + [[nodiscard]] const QueryField &RightFieldData() const &noexcept { return rightField_; } + [[nodiscard]] QueryField &RightFieldData() &noexcept { return rightField_; } + [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return leftField_.FieldsHaveBeenSet() && rightField_.FieldsHaveBeenSet(); } + [[nodiscard]] bool IsLeftFieldIndexed() const noexcept { return leftField_.IsFieldIndexed(); } + [[nodiscard]] bool IsRightFieldIndexed() const noexcept { return rightField_.IsFieldIndexed(); } + [[nodiscard]] std::string Dump() const; + + auto LeftFieldName() const && = delete; + auto RightFieldName() const && = delete; + auto LeftFields() const && = delete; + auto RightFields() const && = delete; + auto LeftCompositeFieldsTypes() const && = delete; + auto RightCompositeFieldsTypes() const && = delete; + auto LeftFieldData() const && = delete; + auto RightFieldData() const && = delete; private: + QueryField leftField_; + QueryField rightField_; CondType condition_; }; @@ -103,9 +241,7 @@ class QueryEntries void ToDsl(const Query &parentQuery, JsonBuilder &builder) const { return toDsl(cbegin(), cend(), parentQuery, builder); } void WriteSQLWhere(const Query &parentQuery, WrSerializer &, bool stripArgs) const; void Serialize(WrSerializer &ser) const { serialize(cbegin(), cend(), ser); } - bool CheckIfSatisfyConditions(const ConstPayload &pl, TagsMatcher &tm) const { - return checkIfSatisfyConditions(cbegin(), cend(), pl, tm); - } + bool CheckIfSatisfyConditions(const ConstPayload &pl) const { return checkIfSatisfyConditions(cbegin(), cend(), pl); } template std::string Dump(const std::vector &joinedSelectors) const { WrSerializer ser; @@ -120,57 +256,22 @@ class QueryEntries static void toDsl(const_iterator it, const_iterator to, const Query &parentQuery, JsonBuilder &); static void writeSQL(const Query &parentQuery, const_iterator from, const_iterator to, WrSerializer &, bool stripArgs); static void serialize(const_iterator it, const_iterator to, WrSerializer &); - static bool checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &, TagsMatcher &); - static bool checkIfSatisfyCondition(const QueryEntry &, const ConstPayload &, TagsMatcher &); - static bool checkIfSatisfyCondition(const BetweenFieldsQueryEntry &, const ConstPayload &, TagsMatcher &); + static bool checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &); + static bool checkIfSatisfyCondition(const QueryEntry &, const ConstPayload &); + static bool checkIfSatisfyCondition(const BetweenFieldsQueryEntry &, const ConstPayload &); static bool checkIfSatisfyCondition(const VariantArray &lValues, CondType, const VariantArray &rValues); protected: - static void dumpEqualPositions(size_t level, WrSerializer &ser, const EqualPositions_t &equalPositions) { - for (const auto &eq : equalPositions) { - for (size_t i = 0; i < level; ++i) { - ser << " "; - } - ser << "equal_poisition("; - for (size_t i = 0, s = eq.size(); i < s; ++i) { - if (i != 0) ser << ", "; - ser << eq[i]; - } - ser << ")\n"; - } - } - + static void dumpEqualPositions(size_t level, WrSerializer &, const EqualPositions_t &); template - static void dump(size_t level, const_iterator begin, const_iterator end, const std::vector &joinedSelectors, WrSerializer &ser) { - for (const_iterator it = begin; it != end; ++it) { - for (size_t i = 0; i < level; ++i) { - ser << " "; - } - if (it != begin || it->operation != OpAnd) { - ser << it->operation << ' '; - } - it->InvokeAppropriate( - [&](const QueryEntriesBracket &b) { - ser << "(\n"; - dump(level + 1, it.cbegin(), it.cend(), joinedSelectors, ser); - dumpEqualPositions(level + 1, ser, b.equalPositions); - for (size_t i = 0; i < level; ++i) { - ser << " "; - } - ser << ")\n"; - }, - [&ser](const QueryEntry &qe) { ser << qe.Dump() << '\n'; }, - [&joinedSelectors, &ser](const JoinQueryEntry &jqe) { ser << jqe.Dump(joinedSelectors) << '\n'; }, - [&ser](const BetweenFieldsQueryEntry &qe) { ser << qe.Dump() << '\n'; }, - [&ser](const AlwaysFalse &) { ser << "AlwaysFalse" << 'n'; }); - } - } + static void dump(size_t level, const_iterator begin, const_iterator end, const std::vector &joinedSelectors, WrSerializer &); }; class UpdateEntry { public: - UpdateEntry(std::string c, VariantArray v, FieldModifyMode m = FieldModeSet, bool e = false) - : column_(std::move(c)), values_(std::move(v)), mode_(m), isExpression_(e) { + template + UpdateEntry(Str &&c, VariantArray &&v, FieldModifyMode m = FieldModeSet, bool e = false) + : column_(std::forward(c)), values_(std::move(v)), mode_(m), isExpression_(e) { if (column_.empty()) { throw Error{errParams, "Empty update column name"}; } @@ -192,36 +293,71 @@ class UpdateEntry { bool isExpression_ = false; }; -struct QueryJoinEntry { - QueryJoinEntry() = default; - QueryJoinEntry(OpType op, CondType cond, std::string idx, std::string jIdx) - : op_{op}, condition_{cond}, index_{std::move(idx)}, joinIndex_{std::move(jIdx)} {} - bool operator==(const QueryJoinEntry &) const noexcept; - bool operator!=(const QueryJoinEntry &qje) const noexcept { return !operator==(qje); } - OpType op_ = OpAnd; - CondType condition_ = CondEq; ///< Condition applied to expression: index_ COND joinIndex_ - std::string index_; ///< main ns index field name - std::string joinIndex_; ///< joining ns index field name - int idxNo = -1; ///< index_ field Index number in main ns - bool reverseNamespacesOrder = false; ///< controls SQL encoding order - ///< false: mainNs.index Condition joinNs.joinIndex - ///< true: joinNs.joinIndex Invert(Condition) mainNs.index +class QueryJoinEntry { +public: + QueryJoinEntry(OpType op, CondType cond, std::string &&leftFld, std::string &&rightFld, bool reverseNs = false) noexcept + : leftField_{std::move(leftFld)}, rightField_{std::move(rightFld)}, op_{op}, condition_{cond}, reverseNamespacesOrder_{reverseNs} {} + [[nodiscard]] bool operator==(const QueryJoinEntry &) const noexcept; + [[nodiscard]] bool operator!=(const QueryJoinEntry &other) const noexcept { return !operator==(other); } + [[nodiscard]] bool IsLeftFieldIndexed() const noexcept { return leftField_.IsFieldIndexed(); } + [[nodiscard]] bool IsRightFieldIndexed() const noexcept { return rightField_.IsFieldIndexed(); } + [[nodiscard]] int LeftIdxNo() const noexcept { return leftField_.IndexNo(); } + [[nodiscard]] int RightIdxNo() const noexcept { return rightField_.IndexNo(); } + [[nodiscard]] const FieldsSet &LeftFields() const &noexcept { return leftField_.Fields(); } + [[nodiscard]] const FieldsSet &RightFields() const &noexcept { return rightField_.Fields(); } + [[nodiscard]] KeyValueType LeftFieldType() const noexcept { return leftField_.FieldType(); } + [[nodiscard]] KeyValueType RightFieldType() const noexcept { return rightField_.FieldType(); } + [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const &noexcept { return leftField_.CompositeFieldsTypes(); } + [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const &noexcept { + return rightField_.CompositeFieldsTypes(); + } + [[nodiscard]] OpType Operation() const noexcept { return op_; } + [[nodiscard]] CondType Condition() const noexcept { return condition_; } + [[nodiscard]] const std::string &LeftFieldName() const &noexcept { return leftField_.FieldName(); } + [[nodiscard]] const std::string &RightFieldName() const &noexcept { return rightField_.FieldName(); } + [[nodiscard]] bool ReverseNamespacesOrder() const noexcept { return reverseNamespacesOrder_; } + [[nodiscard]] const QueryField &LeftFieldData() const &noexcept { return leftField_; } + [[nodiscard]] QueryField &LeftFieldData() &noexcept { return leftField_; } + [[nodiscard]] const QueryField &RightFieldData() const &noexcept { return rightField_; } + [[nodiscard]] QueryField &RightFieldData() &noexcept { return rightField_; } + void SetLeftIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) & { + leftField_.SetIndexData(idxNo, std::move(fields), fieldType, selectType, std::move(compositeFieldsTypes)); + } + void SetRightIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) & { + rightField_.SetIndexData(idxNo, std::move(fields), fieldType, selectType, std::move(compositeFieldsTypes)); + } + void SetLeftField(FieldsSet &&fields) & { leftField_.SetField(std::move(fields)); } + void SetRightField(FieldsSet &&fields) & { rightField_.SetField(std::move(fields)); } + [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return leftField_.FieldsHaveBeenSet() && rightField_.FieldsHaveBeenSet(); } template - std::string DumpCondition(const JS &joinedSelector, bool needOp = false) const { - WrSerializer ser; - const auto &q = joinedSelector.JoinQuery(); - if (needOp) { - ser << ' ' << op_ << ' '; - } - ser << q._namespace << '.' << joinIndex_ << ' ' << InvertJoinCondition(condition_) << ' ' << index_; - return std::string{ser.Slice()}; - } + std::string DumpCondition(const JS &joinedSelector, bool needOp = false) const; + + auto LeftFields() const && = delete; + auto RightFields() const && = delete; + auto LeftCompositeFieldsTypes() const && = delete; + auto RightCompositeFieldsTypes() const && = delete; + auto LeftFieldName() const && = delete; + auto RightFieldName() const && = delete; + auto LeftFieldData() const && = delete; + auto RightFieldData() const && = delete; + +private: + QueryField leftField_; + QueryField rightField_; + const OpType op_; + const CondType condition_; + const bool reverseNamespacesOrder_; ///< controls SQL encoding order + ///< false: mainNs.index Condition joinNs.joinIndex + ///< true: joinNs.joinIndex Invert(Condition) mainNs.index }; struct SortingEntry { SortingEntry() noexcept = default; - SortingEntry(std::string e, bool d) noexcept : expression(std::move(e)), desc(d) {} + template + SortingEntry(Str &&e, bool d) noexcept : expression(std::forward(e)), desc(d) {} bool operator==(const SortingEntry &) const noexcept; bool operator!=(const SortingEntry &se) const noexcept { return !operator==(se); } std::string expression; @@ -233,7 +369,7 @@ struct SortingEntries : public h_vector {}; class AggregateEntry { public: - AggregateEntry(AggType type, h_vector fields, SortingEntries sort = {}, unsigned limit = QueryEntry::kDefaultLimit, + AggregateEntry(AggType type, h_vector &&fields, SortingEntries &&sort = {}, unsigned limit = QueryEntry::kDefaultLimit, unsigned offset = QueryEntry::kDefaultOffset); [[nodiscard]] bool operator==(const AggregateEntry &) const noexcept; [[nodiscard]] bool operator!=(const AggregateEntry &ae) const noexcept { return !operator==(ae); } @@ -242,7 +378,7 @@ class AggregateEntry { [[nodiscard]] const SortingEntries &Sorting() const noexcept { return sortingEntries_; } [[nodiscard]] unsigned Limit() const noexcept { return limit_; } [[nodiscard]] unsigned Offset() const noexcept { return offset_; } - void AddSortingEntry(SortingEntry); + void AddSortingEntry(SortingEntry &&); void SetLimit(unsigned); void SetOffset(unsigned); diff --git a/cpp_src/core/query/sql/sqlencoder.cc b/cpp_src/core/query/sql/sqlencoder.cc index aa82c1110..020ffea20 100644 --- a/cpp_src/core/query/sql/sqlencoder.cc +++ b/cpp_src/core/query/sql/sqlencoder.cc @@ -1,7 +1,7 @@ #include "core/query/sql/sqlencoder.h" #include "core/keyvalue/geometry.h" -#include "core/nsselecter/sortexpression.h" +#include "core/keyvalue/p_string.h" #include "core/queryresults/aggregationresult.h" #include "core/type_consts_helpers.h" #include "tools/serializer.h" @@ -16,7 +16,7 @@ static void indexToSql(const std::string &index, WrSerializer &ser) { } } -static WrSerializer &stringToSql(const std::string &str, WrSerializer &ser) { +static WrSerializer &stringToSql(std::string_view str, WrSerializer &ser) { ser << '\''; for (auto c : str) { switch (c) { @@ -48,15 +48,12 @@ static WrSerializer &stringToSql(const std::string &str, WrSerializer &ser) { ser << '\''; return ser; } - -SQLEncoder::SQLEncoder(const Query &q) : query_(q) {} - void SQLEncoder::DumpSingleJoinQuery(size_t idx, WrSerializer &ser, bool stripArgs) const { assertrx(idx < query_.joinQueries_.size()); const auto &jq = query_.joinQueries_[idx]; ser << ' ' << jq.joinType; - if (jq.entries.Empty() && jq.count == QueryEntry::kDefaultLimit && jq.sortingEntries_.empty()) { - ser << ' ' << jq._namespace << " ON "; + if (jq.entries.Empty() && !jq.HasLimit() && jq.sortingEntries_.empty()) { + ser << ' ' << jq.NsName() << " ON "; } else { ser << " ("; jq.GetSQL(ser, stripArgs); @@ -65,13 +62,13 @@ void SQLEncoder::DumpSingleJoinQuery(size_t idx, WrSerializer &ser, bool stripAr if (jq.joinEntries_.size() != 1) ser << "("; for (auto &e : jq.joinEntries_) { if (&e != &*jq.joinEntries_.begin()) { - ser << ' ' << e.op_ << ' '; + ser << ' ' << e.Operation() << ' '; } - if (e.reverseNamespacesOrder) { - ser << jq._namespace << '.' << e.joinIndex_ << ' ' << InvertJoinCondition(e.condition_) << ' ' << query_._namespace << '.' - << e.index_; + if (e.ReverseNamespacesOrder()) { + ser << jq.NsName() << '.' << e.RightFieldName() << ' ' << InvertJoinCondition(e.Condition()) << ' ' << query_.NsName() << '.' + << e.LeftFieldName(); } else { - ser << query_._namespace << '.' << e.index_ << ' ' << e.condition_ << ' ' << jq._namespace << '.' << e.joinIndex_; + ser << query_.NsName() << '.' << e.LeftFieldName() << ' ' << e.Condition() << ' ' << jq.NsName() << '.' << e.RightFieldName(); } } if (jq.joinEntries_.size() != 1) ser << ')'; @@ -137,7 +134,7 @@ void SQLEncoder::dumpEqualPositions(WrSerializer &ser, const EqualPositions_t &e } WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { - switch (query_.type_) { + switch (realQueryType_) { case QuerySelect: { ser << "SELECT "; bool needComma = false; @@ -171,10 +168,10 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { distinctIndex = query_.aggregations_[0].Fields()[0]; } if (query_.selectFilter_.empty()) { - if (query_.count != 0 || query_.calcTotal == ModeNoTotal) { + if (query_.Limit() != 0 || query_.CalcTotal() == ModeNoTotal) { if (needComma) ser << ", "; ser << '*'; - if (query_.calcTotal != ModeNoTotal) { + if (query_.CalcTotal() != ModeNoTotal) { needComma = true; } } @@ -190,19 +187,19 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { } } } - if (query_.calcTotal != ModeNoTotal) { + if (query_.CalcTotal() != ModeNoTotal) { if (needComma) ser << ", "; - if (query_.calcTotal == ModeAccurateTotal) ser << "COUNT(*)"; - if (query_.calcTotal == ModeCachedTotal) ser << "COUNT_CACHED(*)"; + if (query_.CalcTotal() == ModeAccurateTotal) ser << "COUNT(*)"; + if (query_.CalcTotal() == ModeCachedTotal) ser << "COUNT_CACHED(*)"; } - ser << " FROM " << query_._namespace; + ser << " FROM " << query_.NsName(); } break; case QueryDelete: - ser << "DELETE FROM " << query_._namespace; + ser << "DELETE FROM " << query_.NsName(); break; case QueryUpdate: { if (query_.UpdateFields().empty()) break; - ser << "UPDATE " << query_._namespace; + ser << "UPDATE " << query_.NsName(); FieldModifyMode mode = query_.UpdateFields().front().Mode(); bool isUpdate = (mode == FieldModeSet || mode == FieldModeSetJson); if (isUpdate) { @@ -219,11 +216,19 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { if (isArray) ser << '['; for (const Variant &v : field.Values()) { if (&v != &*field.Values().begin()) ser << ','; - if (v.Type().Is() && !field.IsExpression() && (mode != FieldModeSetJson)) { - stringToSql(v.As(), ser); - } else { - ser << v.As(); - } + v.Type().EvaluateOneOf(overloaded{ + [&](KeyValueType::String) { + if (!field.IsExpression() && mode != FieldModeSetJson) { + stringToSql(v.As(), ser); + } else { + ser << v.As(); + } + }, + [&](KeyValueType::Uuid) { ser << '\'' << v.As() << '\''; }, + [&](OneOf) { + ser << v.As(); + }}); } if (isArray) ser << "]"; } @@ -231,7 +236,7 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { break; } case QueryTruncate: - ser << "TRUNCATE " << query_._namespace; + ser << "TRUNCATE " << query_.NsName(); break; default: throw Error(errParams, "Not implemented"); @@ -242,8 +247,8 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { dumpMerged(ser, stripArgs); dumpOrderBy(ser, stripArgs); - if (query_.start != QueryEntry::kDefaultOffset && !stripArgs) ser << " OFFSET " << query_.start; - if (query_.count != QueryEntry::kDefaultLimit && !stripArgs) ser << " LIMIT " << query_.count; + if (query_.HasOffset() && !stripArgs) ser << " OFFSET " << query_.Offset(); + if (query_.HasLimit() && !stripArgs) ser << " LIMIT " << query_.Limit(); return ser; } @@ -274,41 +279,43 @@ void SQLEncoder::dumpWhereEntries(QueryEntries::const_iterator from, QueryEntrie if (encodedEntries) { ser << opNames[op] << ' '; } - if (entry.condition == CondDWithin) { + if (entry.Condition() == CondDWithin) { ser << "ST_DWithin("; - indexToSql(entry.index, ser); + indexToSql(entry.FieldName(), ser); if (stripArgs) { ser << ", ?, ?)"; } else { - assertrx(entry.values.size() == 2); + assertrx(entry.Values().size() == 2); Point point; double distance; - if (entry.values[0].Type().Is()) { - point = static_cast(entry.values[0]); - distance = entry.values[1].As(); + if (entry.Values()[0].Type().Is()) { + point = static_cast(entry.Values()[0]); + distance = entry.Values()[1].As(); } else { - point = static_cast(entry.values[1]); - distance = entry.values[0].As(); + point = static_cast(entry.Values()[1]); + distance = entry.Values()[0].As(); } ser << ", ST_GeomFromText('POINT(" << point.X() << ' ' << point.Y() << ")'), " << distance << ')'; } } else { - indexToSql(entry.index, ser); - ser << ' ' << entry.condition << ' '; - if (entry.condition == CondEmpty || entry.condition == CondAny) { + indexToSql(entry.FieldName(), ser); + ser << ' ' << entry.Condition() << ' '; + if (entry.Condition() == CondEmpty || entry.Condition() == CondAny) { } else if (stripArgs) { ser << '?'; } else { - if (entry.values.size() != 1) ser << '('; - for (auto &v : entry.values) { - if (&v != &entry.values[0]) ser << ','; - if (v.Type().Is()) { - stringToSql(v.As(), ser); - } else { - ser << v.As(); - } + if (entry.Values().size() != 1) ser << '('; + for (auto &v : entry.Values()) { + if (&v != &entry.Values()[0]) ser << ','; + v.Type().EvaluateOneOf(overloaded{ + [&](KeyValueType::String) { stringToSql(v.As(), ser); }, + [&](KeyValueType::Uuid) { ser << '\'' << v.As() << '\''; }, + [&](OneOf) { + ser << v.As(); + }}); } - if (entry.values.size() != 1) ser << ")"; + if (entry.Values().size() != 1) ser << ")"; } } }, @@ -319,9 +326,9 @@ void SQLEncoder::dumpWhereEntries(QueryEntries::const_iterator from, QueryEntrie SQLEncoder(query_).DumpSingleJoinQuery(jqe.joinIndex, ser, stripArgs); }, [&ser](const BetweenFieldsQueryEntry &entry) { - indexToSql(entry.firstIndex, ser); + indexToSql(entry.LeftFieldName(), ser); ser << ' ' << entry.Condition() << ' '; - indexToSql(entry.secondIndex, ser); + indexToSql(entry.RightFieldName(), ser); }); ++encodedEntries; } diff --git a/cpp_src/core/query/sql/sqlencoder.h b/cpp_src/core/query/sql/sqlencoder.h index c1b217609..968f387df 100644 --- a/cpp_src/core/query/sql/sqlencoder.h +++ b/cpp_src/core/query/sql/sqlencoder.h @@ -2,7 +2,6 @@ #include #include "core/query/query.h" -#include "core/type_consts.h" /// @namespace reindexer /// The base namespace @@ -12,7 +11,8 @@ class WrSerializer; class SQLEncoder { public: - SQLEncoder(const Query &q); + SQLEncoder(const Query &q) noexcept : SQLEncoder(q, q.Type()) {} + SQLEncoder(const Query &q, QueryType queryType) noexcept : query_(q), realQueryType_(queryType) {} WrSerializer &GetSQL(WrSerializer &ser, bool stripArgs = false) const; @@ -40,8 +40,8 @@ class SQLEncoder { /// Builds a print version of all equal_position() functions in query. /// @param ser - serializer to store SQL string - /// @param parenthesisIndex - index of current parenthesis - void dumpEqualPositions(WrSerializer &ser, const EqualPositions_t &) const; + /// @param equalPositions - equal positions array + void dumpEqualPositions(WrSerializer &ser, const EqualPositions_t &equalPositions) const; /// Builds a print version of all where condition entries. /// @param from - iterator to first entry @@ -52,6 +52,7 @@ class SQLEncoder { void dumpSQLWhere(WrSerializer &ser, bool stripArgs) const; const Query &query_; + const QueryType realQueryType_; }; } // namespace reindexer diff --git a/cpp_src/core/query/sql/sqlparser.cc b/cpp_src/core/query/sql/sqlparser.cc index 8549d03ae..1cb96cfd1 100644 --- a/cpp_src/core/query/sql/sqlparser.cc +++ b/cpp_src/core/query/sql/sqlparser.cc @@ -6,6 +6,7 @@ #include "core/queryresults/aggregationresult.h" #include "core/type_consts_helpers.h" #include "sqltokentype.h" +#include "tools/stringstools.h" #include "vendor/double-conversion/double-conversion.h" #include "vendor/gason/gason.h" @@ -91,16 +92,16 @@ int SQLParser::selectParse(tokenizer &parser) { parser.next_token(); tok = peekSqlToken(parser, SingleSelectFieldSqlToken); if (name.text() == "count"sv) { - query_.calcTotal = ModeAccurateTotal; + query_.CalcTotal(ModeAccurateTotal); if (!wasSelectFilter) { - query_.count = 0; + query_.Limit(0); } tok = parser.next_token(); if (tok.text() != "*") throw Error(errParseSQL, "Expected '*', but found '%s' in query, %s", tok.text(), parser.where()); } else if (name.text() == "count_cached"sv) { - query_.calcTotal = ModeCachedTotal; + query_.CalcTotal(ModeCachedTotal); if (!wasSelectFilter) { - query_.count = 0; + query_.Limit(0); } tok = parser.next_token(); if (tok.text() != "*"sv) throw Error(errParseSQL, "Expected '*', but found '%s' in query, %s", tok.text(), parser.where()); @@ -168,13 +169,13 @@ int SQLParser::selectParse(tokenizer &parser) { throw Error(errConflict, kAggregationWithSelectFieldsMsgError); } query_.selectFilter_.emplace_back(nameWithCase.text()); - query_.count = QueryEntry::kDefaultLimit; + query_.Limit(QueryEntry::kDefaultLimit); wasSelectFilter = true; } else if (name.text() == "*"sv) { if (!query_.CanAddSelectFilter()) { throw Error(errConflict, kAggregationWithSelectFieldsMsgError); } - query_.count = QueryEntry::kDefaultLimit; + query_.Limit(QueryEntry::kDefaultLimit); wasSelectFilter = true; query_.selectFilter_.clear(); } @@ -187,8 +188,8 @@ int SQLParser::selectParse(tokenizer &parser) { throw Error(errParams, "Expected 'FROM', but found '%s' in query, %s", tok.text(), parser.where()); peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(parser.next_token().text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(parser.next_token().text()); + ctx_.updateLinkedNs(query_.NsName()); while (!parser.end()) { tok = peekSqlToken(parser, SelectConditionsStart); @@ -200,17 +201,17 @@ int SQLParser::selectParse(tokenizer &parser) { tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.count = stoi(tok.text()); + query_.Limit(stoi(tok.text())); } else if (tok.text() == "offset"sv) { parser.next_token(); tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.start = stoi(tok.text()); + query_.Offset(stoi(tok.text())); } else if (tok.text() == "order"sv) { parser.next_token(); parseOrderBy(parser, query_.sortingEntries_, query_.forcedSortOrder_); - ctx_.updateLinkedNs(query_._namespace); + ctx_.updateLinkedNs(query_.NsName()); } else if (tok.text() == "join"sv) { parser.next_token(); parseJoin(JoinType::LeftJoin, parser); @@ -402,8 +403,8 @@ int SQLParser::deleteParse(tokenizer &parser) { throw Error(errParams, "Expected 'FROM', but found '%s' in query, %s", tok.text(), parser.where()); peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(parser.next_token().text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(parser.next_token().text()); + ctx_.updateLinkedNs(query_.NsName()); while (!parser.end()) { tok = peekSqlToken(parser, DeleteConditionsStart); @@ -415,17 +416,17 @@ int SQLParser::deleteParse(tokenizer &parser) { tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.count = stoi(tok.text()); + query_.Limit(stoi(tok.text())); } else if (tok.text() == "offset"sv) { parser.next_token(); tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.start = stoi(tok.text()); + query_.Offset(stoi(tok.text())); } else if (tok.text() == "order"sv) { parser.next_token(); parseOrderBy(parser, query_.sortingEntries_, query_.forcedSortOrder_); - ctx_.updateLinkedNs(query_._namespace); + ctx_.updateLinkedNs(query_.NsName()); } else break; } @@ -491,7 +492,7 @@ UpdateEntry SQLParser::parseUpdateField(tokenizer &parser) { if (tok.type != TokenName) { throw Error(errParseSQL, "Expected field name but found '%s' in query %s", tok.text(), parser.where()); } - UpdateEntry updateField{{tok.text().data(), tok.text().length()}, {}}; + UpdateEntry updateField{tok.text(), {}}; parser.next_token(); tok = parser.next_token(); @@ -545,8 +546,8 @@ int SQLParser::updateParse(tokenizer &parser) { parser.next_token(); token tok = peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(tok.text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(tok.text()); + ctx_.updateLinkedNs(query_.NsName()); parser.next_token(); tok = peekSqlToken(parser, UpdateOptionsSqlToken); @@ -586,8 +587,8 @@ int SQLParser::updateParse(tokenizer &parser) { int SQLParser::truncateParse(tokenizer &parser) { parser.next_token(); token tok = peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(tok.text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(tok.text()); + ctx_.updateLinkedNs(query_.NsName()); parser.next_token(); return 0; } @@ -650,7 +651,7 @@ int SQLParser::parseWhere(tokenizer &parser) { nextOp = OpAnd; } else { // Index name - const std::string index{tok.text()}; + std::string index{tok.text()}; // Operator CondType condition; @@ -673,13 +674,13 @@ int SQLParser::parseWhere(tokenizer &parser) { if (ctx_.autocompleteMode) peekSqlToken(parser, WhereFieldValueSqlToken, false); tok = parser.next_token(); if (iequals(tok.text(), "null"sv) || iequals(tok.text(), "empty"sv)) { - query_.entries.Append(nextOp, QueryEntry{index, CondEmpty, {}}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), CondEmpty, {}}); } else if (iequals(tok.text(), "not"sv)) { tok = peekSqlToken(parser, WhereFieldNegateValueSqlToken, false); if (!iequals(tok.text(), "null"sv) && !iequals(tok.text(), "empty"sv)) { throw Error(errParseSQL, "Expected NULL, but found '%s' in query, %s", tok.text(), parser.where()); } - query_.entries.Append(nextOp, QueryEntry{index, CondAny, {}}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), CondAny, {}}); tok = parser.next_token(false); } else if (tok.text() == "("sv) { VariantArray values; @@ -692,12 +693,12 @@ int SQLParser::parseWhere(tokenizer &parser) { if (tok.text() != ","sv) throw Error(errParseSQL, "Expected ')' or ',', but found '%s' in query, %s", tok.text(), parser.where()); } - query_.entries.Append(nextOp, QueryEntry{index, condition, std::move(values)}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), condition, std::move(values)}); } else if (tok.type != TokenName || toLower(tok.text()) == "true" || toLower(tok.text()) == "false") { - query_.entries.Append(nextOp, QueryEntry{index, condition, {token2kv(tok, parser, true)}}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), condition, {token2kv(tok, parser, true)}}); // Second field } else { - query_.entries.Append(nextOp, BetweenFieldsQueryEntry{index, condition, std::string{tok.text()}}); + query_.entries.Append(nextOp, BetweenFieldsQueryEntry{std::move(index), condition, std::string{tok.text()}}); } nextOp = OpAnd; } @@ -766,7 +767,7 @@ void SQLParser::parseEqualPositions(tokenizer &parser, std::vectorHoldsOrReferTo() && nameWithCase.text() == it->Value().index) { + if (it->HoldsOrReferTo() && nameWithCase.text() == it->Value().FieldName()) { validField = true; break; } @@ -933,11 +934,11 @@ void SQLParser::parseJoin(JoinType type, tokenizer &parser) { throw Error(errParseSQL, "Expected ')', but found %s, %s", tok.text(), parser.where()); } } else { - jquery._namespace = std::string(tok.text()); - ctx_.updateLinkedNs(jquery._namespace); + jquery.SetNsName(tok.text()); + ctx_.updateLinkedNs(jquery.NsName()); } jquery.joinType = type; - jparser.parseJoinEntries(parser, query_._namespace, jquery); + jparser.parseJoinEntries(parser, query_.NsName(), jquery); if (type != JoinType::LeftJoin) { query_.entries.Append((type == JoinType::InnerJoin) ? OpAnd : OpOr, JoinQueryEntry(query_.joinQueries_.size())); @@ -1023,28 +1024,25 @@ void SQLParser::parseJoinEntries(tokenizer &parser, const std::string &mainNs, J return; } - QueryJoinEntry je; - std::string ns1 = mainNs, ns2 = jquery._namespace; - std::string idx1 = parseJoinedFieldName(parser, ns1); - je.condition_ = getCondType(parser.next_token().text()); - std::string idx2 = parseJoinedFieldName(parser, ns2); - - if (ns1 == mainNs && ns2 == jquery._namespace) { - je.index_ = std::move(idx1); - je.joinIndex_ = std::move(idx2); - } else if (ns2 == mainNs && ns1 == jquery._namespace) { - je.index_ = std::move(idx2); - je.joinIndex_ = std::move(idx1); - je.condition_ = InvertJoinCondition(je.condition_); - je.reverseNamespacesOrder = true; - } else { - throw Error(errParseSQL, "Unexpected tables with ON statement: ('%s' and '%s') but expected ('%s' and '%s'), %s", ns1, ns2, - mainNs, jquery._namespace, parser.where()); + std::string ns1 = mainNs, ns2 = jquery.NsName(); + std::string fld1 = parseJoinedFieldName(parser, ns1); + CondType condition = getCondType(parser.next_token().text()); + std::string fld2 = parseJoinedFieldName(parser, ns2); + bool reverseNamespacesOrder{false}; + + if (ns1 != mainNs || ns2 != jquery.NsName()) { + if (ns2 == mainNs && ns1 == jquery.NsName()) { + std::swap(fld1, fld2); + condition = InvertJoinCondition(condition); + reverseNamespacesOrder = true; + } else { + throw Error(errParseSQL, "Unexpected tables with ON statement: ('%s' and '%s') but expected ('%s' and '%s'), %s", ns1, ns2, + mainNs, jquery.NsName(), parser.where()); + } } - je.op_ = jquery.nextOp_; + jquery.joinEntries_.emplace_back(jquery.nextOp_, condition, std::move(fld1), std::move(fld2), reverseNamespacesOrder); jquery.nextOp_ = OpAnd; - jquery.joinEntries_.emplace_back(std::move(je)); if (!braces) { return; } diff --git a/cpp_src/core/query/sql/sqlsuggester.h b/cpp_src/core/query/sql/sqlsuggester.h index f0e0feebf..e0d55ec29 100644 --- a/cpp_src/core/query/sql/sqlsuggester.h +++ b/cpp_src/core/query/sql/sqlsuggester.h @@ -2,9 +2,7 @@ #include #include "core/schema.h" -#include "estl/fast_hash_map.h" #include "sqlparser.h" -#include "tools/stringstools.h" namespace reindexer { diff --git a/cpp_src/core/querycache.h b/cpp_src/core/querycache.h index 4d98bd1d2..a9c0c4c14 100644 --- a/cpp_src/core/querycache.h +++ b/cpp_src/core/querycache.h @@ -8,9 +8,9 @@ namespace reindexer { -struct QueryTotalCountCacheVal { - QueryTotalCountCacheVal() = default; - QueryTotalCountCacheVal(size_t total) noexcept : total_count(total) {} +struct QueryCountCacheVal { + QueryCountCacheVal() = default; + QueryCountCacheVal(size_t total) noexcept : total_count(total) {} size_t Size() const noexcept { return 0; } @@ -49,9 +49,8 @@ struct HashQueryCacheKey { } }; -struct QueryTotalCountCache : LRUCache { - QueryTotalCountCache(size_t sizeLimit = kDefaultCacheSizeLimit, int hitCount = kDefaultHitCountToCache) - : LRUCache(sizeLimit, hitCount) {} -}; +using QueryCountCache = LRUCache; + +; } // namespace reindexer diff --git a/cpp_src/core/queryresults/queryresults.cc b/cpp_src/core/queryresults/queryresults.cc index 1ba73eaec..a17df13d5 100644 --- a/cpp_src/core/queryresults/queryresults.cc +++ b/cpp_src/core/queryresults/queryresults.cc @@ -427,18 +427,6 @@ Item QueryResults::Iterator::GetItem(bool enableHold) { return item; } -QueryResults::Iterator &QueryResults::Iterator::operator++() { - idx_++; - return *this; -} -QueryResults::Iterator &QueryResults::Iterator::operator+(int val) { - idx_ += val; - return *this; -} - -bool QueryResults::Iterator::operator!=(const Iterator &other) const { return idx_ != other.idx_; } -bool QueryResults::Iterator::operator==(const Iterator &other) const { return idx_ == other.idx_; } - void QueryResults::AddItem(Item &item, bool withData, bool enableHold) { auto ritem = item.impl_; if (item.GetID() != -1) { diff --git a/cpp_src/core/queryresults/queryresults.h b/cpp_src/core/queryresults/queryresults.h index e3e4d11f1..4c06d9d5f 100644 --- a/cpp_src/core/queryresults/queryresults.h +++ b/cpp_src/core/queryresults/queryresults.h @@ -52,10 +52,12 @@ class QueryResults { void AddItem(Item &item, bool withData = false, bool enableHold = true); std::string Dump() const; void Erase(ItemRefVector::iterator begin, ItemRefVector::iterator end); - size_t Count() const { return items_.size(); } - size_t TotalCount() const { return totalCount; } - const std::string &GetExplainResults() const { return explainResults; } - const std::vector &GetAggregationResults() const { return aggregationResults; } + size_t Count() const noexcept { return items_.size(); } + size_t TotalCount() const noexcept { return totalCount; } + const std::string &GetExplainResults() const &noexcept { return explainResults; } + const std::string &GetExplainResults() const && = delete; + const std::vector &GetAggregationResults() const &noexcept { return aggregationResults; } + const std::vector &GetAggregationResults() const && = delete; void Clear(); h_vector GetNamespaces() const; bool IsCacheEnabled() const { return !nonCacheableData; } @@ -77,12 +79,19 @@ class QueryResults { int64_t GetLSN() const { return qr_->items_[idx_].Value().GetLSN(); } bool IsRaw() const; std::string_view GetRaw() const; - Iterator &operator++(); - Iterator &operator+(int delta); - const Error &Status() const noexcept { return err_; } - bool operator!=(const Iterator &) const; - bool operator==(const Iterator &) const; - Iterator &operator*() { return *this; } + Iterator &operator++() noexcept { + idx_++; + return *this; + } + Iterator &operator+(int delta) noexcept { + idx_ += delta; + return *this; + } + + Error Status() const noexcept { return err_; } + bool operator==(const Iterator &other) const noexcept { return idx_ == other.idx_; } + bool operator!=(const Iterator &other) const noexcept { return !operator==(other); } + Iterator &operator*() noexcept { return *this; } const QueryResults *qr_; int idx_; diff --git a/cpp_src/core/queryresults/tableviewbuilder.h b/cpp_src/core/queryresults/tableviewbuilder.h index d9d092127..52fa3247a 100644 --- a/cpp_src/core/queryresults/tableviewbuilder.h +++ b/cpp_src/core/queryresults/tableviewbuilder.h @@ -2,15 +2,13 @@ #include #include +#include #include #include #include #include -#include "tools/errors.h" #include "tools/terminalutils.h" -#include - namespace reindexer { struct ColumnData { diff --git a/cpp_src/core/rdxcontext.h b/cpp_src/core/rdxcontext.h index c79b1eaed..ac7cfce47 100644 --- a/cpp_src/core/rdxcontext.h +++ b/cpp_src/core/rdxcontext.h @@ -18,22 +18,24 @@ struct IRdxCancelContext { virtual ~IRdxCancelContext() = default; }; +constexpr std::string_view kDefaultTimeoutError = "Context timeout"; +constexpr std::string_view kDefaultCancelError = "Context was canceled"; + template -void ThrowOnCancel(const Context& ctx, std::string_view errMsg = std::string_view()) { // TODO may be ""sv +void ThrowOnCancel(const Context& ctx, std::string_view errMsg = std::string_view()) { if (!ctx.isCancelable()) return; - auto cancel = ctx.checkCancel(); + const auto cancel = ctx.checkCancel(); switch (cancel) { case CancelType::Explicit: - throw Error(errCanceled, errMsg); + throw Error(errCanceled, errMsg.empty() ? kDefaultCancelError : errMsg); case CancelType::Timeout: - throw Error(errTimeout, errMsg); + throw Error(errTimeout, errMsg.empty() ? kDefaultTimeoutError : errMsg); case CancelType::None: return; - default: - assertrx(false); - throw Error(errCanceled, errMsg); } + assertrx(false); + throw Error(errCanceled, errMsg.empty() ? kDefaultCancelError : errMsg); } class RdxDeadlineContext : public IRdxCancelContext { @@ -156,7 +158,7 @@ class InternalRdxContext { std::move(user), connectionId); } InternalRdxContext WithContextParams(milliseconds timeout, std::string_view activityTracer, std::string&& user, - int connectionId) const { + int connectionId) const { return activityTracer.empty() ? InternalRdxContext(cmpl_, RdxDeadlineContext(timeout, deadlineCtx_.parent()), activityTracer_, user_, connectionId_) : InternalRdxContext(cmpl_, RdxDeadlineContext(timeout, deadlineCtx_.parent()), diff --git a/cpp_src/core/reindexerimpl.cc b/cpp_src/core/reindexerimpl.cc index b34c8778b..59e1f48bd 100644 --- a/cpp_src/core/reindexerimpl.cc +++ b/cpp_src/core/reindexerimpl.cc @@ -4,17 +4,16 @@ #include #include #include "cjson/jsonbuilder.h" -#include "core/cjson/jsondecoder.h" #include "core/cjson/protobufschemabuilder.h" #include "core/iclientsstats.h" #include "core/index/index.h" #include "core/itemimpl.h" #include "core/nsselecter/crashqueryreporter.h" +#include "core/nsselecter/querypreprocessor.h" #include "core/query/sql/sqlsuggester.h" #include "core/selectfunc/selectfunc.h" #include "core/type_consts_helpers.h" #include "defnsconfigs.h" -#include "estl/contexted_locks.h" #include "estl/defines.h" #include "queryresults/joinresults.h" #include "replicator/replicator.h" @@ -244,7 +243,7 @@ Error ReindexerImpl::Connect(const std::string& dsn, ConnectOpts opts) { idx = nsIdx.fetch_add(1, std::memory_order_relaxed)) { auto& de = foundNs[idx]; if (de.isDir && validateObjectName(de.name, true)) { - if (de.name[0] == '@') { + if (de.name[0] == kTmpNsPrefix) { const std::string tmpPath = fs::JoinPath(storagePath_, de.name); logPrintf(LogWarning, "Dropping tmp namespace '%s'", de.name); if (fs::RmDirAll(tmpPath) < 0) { @@ -336,7 +335,7 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r Namespace::Ptr ns; try { { - ULock lock(mtx_, &rdxCtx); + SLock lock(mtx_, &rdxCtx); if (namespaces_.find(nsDef.name) != namespaces_.end()) { return Error(errParams, "Namespace '%s' already exists", nsDef.name); } @@ -361,7 +360,9 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r namespaces_.insert({nsDef.name, ns}); } if (!nsDef.isTemporary) observers_.OnWALUpdate(LSNPair(), nsDef.name, WALRecord(WalNamespaceAdd)); - for (auto& indexDef : nsDef.indexes) ns->AddIndex(indexDef, rdxCtx); + for (auto& indexDef : nsDef.indexes) { + ns->AddIndex(indexDef, rdxCtx); + } ns->SetSchema(nsDef.schemaJson, rdxCtx); if (nsDef.storage.IsSlaveMode()) ns->setSlaveMode(rdxCtx); @@ -427,7 +428,6 @@ Error ReindexerImpl::closeNamespace(std::string_view nsName, const RdxContext& c try { ULock lock(mtx_, &ctx); auto nsIt = namespaces_.find(nsName); - if (nsIt == namespaces_.end()) { return Error(errNotFound, "Namespace '%s' does not exist", nsName); } @@ -521,7 +521,9 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri const InternalRdxContext& ctx) { Namespace::Ptr dstNs, srcNs; try { - if (dstNsName == srcNsName.data()) return errOK; + if (dstNsName == srcNsName.data()) { + return {}; + } if (isSystemNamespaceNameStrict(srcNsName)) { return Error(errParams, "Can't rename system namespace (%s)", srcNsName); } @@ -536,6 +538,22 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri const auto rdxCtx = ctx.CreateRdxContext( ctx.NeedTraceActivity() ? (ser << "RENAME " << srcNsName << " to " << dstNsName).Slice() : ""sv, activities_); + { + // Perform namespace flushes to minimize chances of the flush under lock + SLock lock(mtx_, &rdxCtx); + auto srcIt = namespaces_.find(srcNsName); + srcNs = (srcIt != namespaces_.end()) ? srcIt->second : Namespace::Ptr(); + lock.unlock(); + + if (srcNs) { + auto err = srcNs->awaitMainNs(rdxCtx)->FlushStorage(rdxCtx); + if (!err.ok()) { + return Error(err.code(), "Unable to flush storage before rename: %s", err.what()); + } + srcNs.reset(); + } + } + ULock lock(mtx_, &rdxCtx); auto srcIt = namespaces_.find(srcNsName); if (srcIt == namespaces_.end()) { @@ -546,8 +564,7 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri auto replState = srcNs->GetReplState(rdxCtx); - if (fromReplication || !replState.slaveMode) // rename from replicator forced temporary ns - { + if (fromReplication || !replState.slaveMode) { // rename from replicator forced temporary ns auto dstIt = namespaces_.find(dstNsName); auto needWalUpdate = !srcNs->GetDefinition(rdxCtx).isTemporary; if (dstIt != namespaces_.end()) { @@ -557,18 +574,16 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri } else { srcNs->Rename(dstNsName, storagePath_, rdxCtx); } - if (needWalUpdate) observers_.OnWALUpdate(LSNPair(), srcNsName, WALRecord(WalNamespaceRename, dstNsName)); - - auto srcNamespace = srcIt->second; namespaces_.erase(srcIt); - namespaces_[dstNsName] = std::move(srcNamespace); + namespaces_[dstNsName] = std::move(srcNs); + if (needWalUpdate) observers_.OnWALUpdate(LSNPair(), srcNsName, WALRecord(WalNamespaceRename, dstNsName)); } else { return Error(errLogic, "Can't rename namespace in slave mode '%s'", srcNsName); } } catch (const Error& err) { return err; } - return errOK; + return {}; } template @@ -678,7 +693,7 @@ Error ReindexerImpl::Update(const Query& q, QueryResults& result, const Internal try { WrSerializer ser; const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? q.GetSQL(ser).Slice() : ""sv, activities_, result); - auto ns = getNamespace(q._namespace, rdxCtx); + auto ns = getNamespace(q.NsName(), rdxCtx); ns->Update(q, result, rdxCtx); if (ns->IsSystem(rdxCtx)) { const std::string kNsName = ns->GetName(rdxCtx); @@ -788,7 +803,7 @@ Error ReindexerImpl::Delete(std::string_view nsName, Item& item, QueryResults& q Error ReindexerImpl::Delete(const Query& q, QueryResults& result, const InternalRdxContext& ctx) { const auto makeCtxStr = [&q](WrSerializer& ser) -> WrSerializer& { return q.GetSQL(ser); }; - const std::string_view nsName = q._namespace; + const std::string_view nsName = q.NsName(); APPLY_NS_FUNCTION2(false, Delete, q, result); } @@ -808,7 +823,7 @@ Error ReindexerImpl::Select(std::string_view query, QueryResults& result, const err = Update(q, result, ctx); break; case QueryTruncate: - err = TruncateNamespace(q._namespace, ctx); + err = TruncateNamespace(q.NsName(), ctx); break; default: err = Error(errParams, "Error unsupported query type %d", q.type_); @@ -840,7 +855,7 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? nonNormalizedSQL.Slice() : "", activities_, result); NsLocker locks(rdxCtx); - auto mainNsWrp = getNamespace(q._namespace, rdxCtx); + auto mainNsWrp = getNamespace(q.NsName(), rdxCtx); auto mainNs = q.IsWALQuery() ? mainNsWrp->awaitMainNs(rdxCtx) : mainNsWrp->getMainNs(); const auto queriesPerfStatsEnabled = configProvider_.QueriesPerfStatsEnabled(); @@ -862,7 +877,7 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal } : std::function{}; - const bool isSystemNsRequest = isSystemNamespaceNameFast(q._namespace); + const bool isSystemNsRequest = isSystemNamespaceNameFast(q.NsName()); QueryStatCalculator statCalculator( std::move(hitter), std::chrono::microseconds(queriesThresholdUS), queriesPerfStatsEnabled || configProvider_.GetSelectLoggingParams().thresholdUs >= 0, @@ -871,13 +886,13 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal StatsLocker::StatsLockT statsSelectLck; if (isSystemNsRequest) { - statsSelectLck = syncSystemNamespaces(q._namespace, detectFilterNsNames(q), rdxCtx); + statsSelectLck = syncSystemNamespaces(q.NsName(), detectFilterNsNames(q), rdxCtx); } // Lookup and lock namespaces_ mainNs->updateSelectTime(); locks.Add(mainNs); q.WalkNested(false, true, [this, &locks, &rdxCtx](const Query& q) { - auto nsWrp = getNamespace(q._namespace, rdxCtx); + auto nsWrp = getNamespace(q.NsName(), rdxCtx); auto ns = q.IsWALQuery() ? nsWrp->awaitMainNs(rdxCtx) : nsWrp->getMainNs(); ns->updateSelectTime(); locks.Add(ns); @@ -955,26 +970,26 @@ bool ReindexerImpl::isPreResultValuesModeOptimizationAvailable(const Query& jIte jItemQ.entries.ExecuteAppropriateForEach( Skip{}, [&jns, &result](const QueryEntry& qe) { - if (qe.idxNo >= 0) { - assertrx(jns->indexes_.size() > static_cast(qe.idxNo)); - const IndexType indexType = jns->indexes_[qe.idxNo]->Type(); + if (qe.IsFieldIndexed()) { + assertrx(jns->indexes_.size() > static_cast(qe.IndexNo())); + const IndexType indexType = jns->indexes_[qe.IndexNo()]->Type(); if (IsComposite(indexType) || IsFullText(indexType)) result = false; } }, [&jns, &result](const BetweenFieldsQueryEntry& qe) { - if (qe.firstIdxNo >= 0) { - assertrx(jns->indexes_.size() > static_cast(qe.firstIdxNo)); - const IndexType indexType = jns->indexes_[qe.firstIdxNo]->Type(); + if (qe.IsLeftFieldIndexed()) { + assertrx(jns->indexes_.size() > static_cast(qe.LeftIdxNo())); + const IndexType indexType = jns->indexes_[qe.LeftIdxNo()]->Type(); if (IsComposite(indexType) || IsFullText(indexType)) result = false; } - if (qe.secondIdxNo >= 0) { - assertrx(jns->indexes_.size() > static_cast(qe.secondIdxNo)); - if (IsComposite(jns->indexes_[qe.secondIdxNo]->Type())) result = false; + if (qe.IsRightFieldIndexed()) { + assertrx(jns->indexes_.size() > static_cast(qe.RightIdxNo())); + if (IsComposite(jns->indexes_[qe.RightIdxNo()]->Type())) result = false; } }); if (!result) return false; for (const auto& se : mainQ.sortingEntries_) { - if (byJoinedField(se.expression, jItemQ._namespace)) return false; // TODO maybe allow #1410 + if (byJoinedField(se.expression, jItemQ.NsName())) return false; // TODO maybe allow #1410 } return true; } @@ -984,14 +999,14 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul std::vector& queryResultsContexts, const RdxContext& rdxCtx) { JoinedSelectors joinedSelectors; if (q.joinQueries_.empty()) return joinedSelectors; - auto ns = locks.Get(q._namespace); + auto ns = locks.Get(q.NsName()); assertrx(ns); // For each joined queries uint32_t joinedSelectorsCount = uint32_t(q.joinQueries_.size()); for (auto& jq : q.joinQueries_) { - if rx_unlikely (isSystemNamespaceNameFast(jq._namespace)) { - throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside JOIN statement", jq._namespace); + if rx_unlikely (isSystemNamespaceNameFast(jq.NsName())) { + throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside JOIN statement", jq.NsName()); } if (rx_unlikely(!jq.joinQueries_.empty())) { throw Error(errParams, "JOINs nested into the other JOINs are not supported"); @@ -1001,13 +1016,13 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul } // Get common results from joined namespaces_ - auto jns = locks.Get(jq._namespace); + auto jns = locks.Get(jq.NsName()); assertrx(jns); // Do join for each item in main result - Query jItemQ(jq._namespace); + Query jItemQ(jq.NsName()); jItemQ.explain_ = q.explain_; - jItemQ.Debug(jq.debugLevel).Limit(jq.count); + jItemQ.Debug(jq.debugLevel).Limit(jq.Limit()); jItemQ.Strict(q.strictMode); for (size_t i = 0; i < jq.sortingEntries_.size(); ++i) { jItemQ.Sort(jq.sortingEntries_[i].expression, jq.sortingEntries_[i].desc); @@ -1017,18 +1032,13 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul // Construct join conditions for (auto& je : jq.joinEntries_) { - int joinIdx = IndexValueType::NotSet; - if (!jns->getIndexByName(je.joinIndex_, joinIdx)) { - joinIdx = IndexValueType::SetByJsonPath; - } - QueryEntry qe(InvertJoinCondition(je.condition_), je.joinIndex_, joinIdx); - if (!ns->getIndexByName(je.index_, const_cast(je).idxNo)) { - const_cast(je).idxNo = IndexValueType::SetByJsonPath; - } - jItemQ.entries.Append(je.op_, std::move(qe)); + QueryPreprocessor::SetQueryField(const_cast(je).LeftFieldData(), *ns); + QueryPreprocessor::SetQueryField(const_cast(je).RightFieldData(), *jns); + jItemQ.entries.Append(je.Operation(), QueryField(je.RightFieldData()), InvertJoinCondition(je.Condition()), + QueryEntry::IgnoreEmptyValues{}); } - Query jjq(jq); + Query jjq(static_cast(jq)); JoinPreResult::Ptr preResult = std::make_shared(); uint32_t joinedFieldIdx = uint32_t(joinedSelectors.size()); JoinCacheRes joinRes; @@ -1058,24 +1068,6 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul result.AddNamespace(jns, true); if (preResult->dataMode == JoinPreResult::ModeValues) { - jItemQ.entries.ExecuteAppropriateForEach( - Skip{}, - [&jns](QueryEntry& qe) { - if (qe.idxNo != IndexValueType::SetByJsonPath) { - assertrx(qe.idxNo >= 0 && static_cast(qe.idxNo) < jns->indexes_.size()); - if (jns->indexes_[qe.idxNo]->Opts().IsSparse()) qe.idxNo = IndexValueType::SetByJsonPath; - } - }, - [&jns](BetweenFieldsQueryEntry& qe) { - if (qe.firstIdxNo != IndexValueType::SetByJsonPath) { - assertrx(qe.firstIdxNo >= 0 && static_cast(qe.firstIdxNo) < jns->indexes_.size()); - if (jns->indexes_[qe.firstIdxNo]->Opts().IsSparse()) qe.firstIdxNo = IndexValueType::SetByJsonPath; - } - if (qe.secondIdxNo != IndexValueType::SetByJsonPath) { - assertrx(qe.secondIdxNo >= 0 && static_cast(qe.secondIdxNo) < jns->indexes_.size()); - if (jns->indexes_[qe.secondIdxNo]->Opts().IsSparse()) qe.secondIdxNo = IndexValueType::SetByJsonPath; - } - }); preResult->values.PreselectAllowed(static_cast(jns->Config().maxPreselectSize) >= preResult->values.size()); if (!preResult->values.Locked()) preResult->values.Lock(); // If not from cache locks.Delete(jns); @@ -1091,9 +1083,9 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul template void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& locks, SelectFunctionsHolder& func, const RdxContext& ctx, QueryStatCalculator& queryStatCalculator) { - auto ns = locks.Get(q._namespace); + auto ns = locks.Get(q.NsName()); if rx_unlikely (!ns) { - throw Error(errParams, "Namespace '%s' does not exist", q._namespace); + throw Error(errParams, "Namespace '%s' does not exist", q.NsName()); } std::vector joinQueryResultsContexts; bool thereAreJoins = !q.joinQueries_.empty(); @@ -1158,7 +1150,7 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& mergeJoinedSelectors.reserve(q.mergeQueries_.size()); uint8_t counter = 0; - auto hasUnsupportedAggreagations = [](const std::vector& aggVector, AggType& t) -> bool { + auto hasUnsupportedAggregations = [](const std::vector& aggVector, AggType& t) -> bool { for (const auto& a : aggVector) { if (a.Type() != AggCount || a.Type() != AggCountCached) { t = a.Type(); @@ -1169,17 +1161,17 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& return false; }; AggType errType; - if (rx_unlikely((q.HasLimit() || q.HasOffset()) && hasUnsupportedAggreagations(q.aggregations_, errType))) { + if (rx_unlikely((q.HasLimit() || q.HasOffset()) && hasUnsupportedAggregations(q.aggregations_, errType))) { throw Error(errParams, "Limit and offset are not supported for aggregations '%s'", AggTypeToStr(errType)); } for (auto& mq : q.mergeQueries_) { - if rx_unlikely (isSystemNamespaceNameFast(mq._namespace)) { - throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside MERGE statement", mq._namespace); + if rx_unlikely (isSystemNamespaceNameFast(mq.NsName())) { + throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside MERGE statement", mq.NsName()); } if rx_unlikely (!mq.sortingEntries_.empty()) { throw Error(errParams, "Sorting in inner merge query is not allowed"); } - if rx_unlikely (!mq.aggregations_.empty() || mq.calcTotal != ModeNoTotal) { + if rx_unlikely (!mq.aggregations_.empty() || mq.CalcTotal() != ModeNoTotal) { throw Error(errParams, "Aggregations in inner merge query is not allowed"); } if rx_unlikely (mq.HasLimit() || mq.HasOffset()) { @@ -1189,7 +1181,7 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& throw Error(errParams, "MERGEs nested into the MERGEs are not supported"); } - auto mns = locks.Get(mq._namespace); + auto mns = locks.Get(mq.NsName()); assertrx(mns); SelectCtx mctx(mq, &q); mctx.nsid = ++counter; @@ -1204,16 +1196,16 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& result.AddNamespace(mns, true); } ItemRefVector& itemRefVec = result.Items(); - if (static_cast(q.start) >= itemRefVec.size()) { + if (q.Offset() >= itemRefVec.size()) { result.Erase(itemRefVec.begin(), itemRefVec.end()); return; } std::sort(itemRefVec.begin(), itemRefVec.end(), ItemRefLess()); - if (q.start > QueryEntry::kDefaultOffset) { - result.Erase(itemRefVec.begin(), itemRefVec.begin() + q.start); + if (q.HasOffset()) { + result.Erase(itemRefVec.begin(), itemRefVec.begin() + q.Offset()); } - if (itemRefVec.size() > q.count) { - result.Erase(itemRefVec.begin() + q.count, itemRefVec.end()); + if (itemRefVec.size() > q.Limit()) { + result.Erase(itemRefVec.begin() + q.Limit(), itemRefVec.end()); } } // Adding context to QueryResults @@ -1234,7 +1226,6 @@ Error ReindexerImpl::Commit(std::string_view /*_namespace*/) { Namespace::Ptr ReindexerImpl::getNamespace(std::string_view nsName, const RdxContext& ctx) { SLock lock(mtx_, &ctx); auto nsIt = namespaces_.find(nsName); - if (nsIt == namespaces_.end()) { throw Error(errParams, "Namespace '%s' does not exist", nsName); } @@ -1645,7 +1636,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) } if (entries.HoldsOrReferTo(i)) { auto& qe = entries.Get(i); - if (qe.index == kNsNameField) { + if (qe.FieldName() == kNsNameField) { if (op == OpNot) { return std::nullopt; } @@ -1653,15 +1644,15 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) [i](const BracketRange& br) noexcept { return i >= br.begin && i < br.end; }) != notBrackets.end()) { return std::nullopt; } - if (qe.condition != CondSet && qe.condition != CondEq) { + if (qe.Condition() != CondSet && qe.Condition() != CondEq) { return std::nullopt; } if (res.has_value()) { return std::nullopt; } res.emplace(); - res->reserve(qe.values.size()); - for (auto& v : qe.values) { + res->reserve(qe.Values().size()); + for (auto& v : qe.Values()) { if (!v.Type().Is()) { return std::nullopt; } @@ -1670,7 +1661,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) } } else if (entries.HoldsOrReferTo(i)) { auto& qe = entries.Get(i); - if (qe.firstIndex == kNsNameField || qe.secondIndex == kNsNameField) { + if (qe.LeftFieldName() == kNsNameField || qe.RightFieldName() == kNsNameField) { return std::nullopt; } } else if (op == OpNot && entries.IsSubTree(i)) { diff --git a/cpp_src/core/schema.cc b/cpp_src/core/schema.cc index 9e9ed51c1..8a43101f8 100644 --- a/cpp_src/core/schema.cc +++ b/cpp_src/core/schema.cc @@ -260,19 +260,13 @@ Error Schema::FromJSON(std::string_view json) { gason::JsonParser parser; auto node = parser.Parse(json); parseJsonNode(node, path, true); - originalJson_.assign(json.data(), json.size()); protobufNsNumber_ = node["x-protobuf-ns-number"].As(-1); - if (protobufNsNumber_ == -1 && originalJson_ != "{}") { + if (protobufNsNumber_ == -1 && json != "{}") { protobufNsNumber_ = counter++; - - // TODO: fix it - auto pos = originalJson_.find_last_of('}'); - if (pos != std::string::npos) { - originalJson_ = originalJson_.erase(pos); - originalJson_ += ",\"x-protobuf-ns-number\":" + std::to_string(protobufNsNumber_) + "}"; - } + originalJson_ = AppendProtobufNumber(json, protobufNsNumber_); + } else { + originalJson_.assign(json); } - } catch (const gason::Exception& ex) { return Error(errParseJson, "Schema: %s\nJson: %s", ex.what(), originalJson_); } catch (const Error& err) { @@ -305,6 +299,18 @@ Error Schema::GetProtobufSchema(WrSerializer& schema) const { return protobufSchemaStatus_; } +std::string Schema::AppendProtobufNumber(std::string_view j, int protobufNsNumber) { + std::string json(j); + if (protobufNsNumber != -1 && j != "{}") { + auto pos = json.find_last_of('}'); + if (pos != std::string::npos) { + json.erase(pos); + json += ",\"x-protobuf-ns-number\":" + std::to_string(protobufNsNumber) + "}"; + } + } + return json; +} + void Schema::parseJsonNode(const gason::JsonNode& node, PrefixTree::PathT& splittedPath, bool isRequired) { bool isArray = false; diff --git a/cpp_src/core/schema.h b/cpp_src/core/schema.h index a28cc3264..aa147f1c7 100644 --- a/cpp_src/core/schema.h +++ b/cpp_src/core/schema.h @@ -133,10 +133,12 @@ class Schema { Error FromJSON(std::string_view json); void GetJSON(WrSerializer&) const; + std::string_view GetJSON() const noexcept { return originalJson_; } Error BuildProtobufSchema(TagsMatcher& tm, PayloadType& pt); Error GetProtobufSchema(WrSerializer& schema) const; int GetProtobufNsNumber() const { return protobufNsNumber_; } const PrefixTree::PrefixTreeNode* GetRoot() const { return &paths_.root_; } + static std::string AppendProtobufNumber(std::string_view j, int protobufNsNumber); std::vector MakeCsvTagOrdering(const TagsMatcher& tm) const; bool IsEmpty() const noexcept; diff --git a/cpp_src/core/selectfunc/nsselectfuncinterface.cc b/cpp_src/core/selectfunc/nsselectfuncinterface.cc index cd4d38e21..ba5c0c8ea 100644 --- a/cpp_src/core/selectfunc/nsselectfuncinterface.cc +++ b/cpp_src/core/selectfunc/nsselectfuncinterface.cc @@ -3,14 +3,16 @@ #include "core/namespace/namespaceimpl.h" namespace reindexer { -const std::string& NsSelectFuncInterface::GetName() const { return nm_.name_; }; -int NsSelectFuncInterface::getIndexByName(const std::string& index) const { return nm_.getIndexByName(index); } -bool NsSelectFuncInterface::getIndexByName(const std::string& name, int& index) const { return nm_.getIndexByName(name, index); } -int NsSelectFuncInterface::getIndexesCount() const { return nm_.indexes_.size(); } +const std::string& NsSelectFuncInterface::GetName() const noexcept { return nm_.name_; }; +int NsSelectFuncInterface::getIndexByName(std::string_view index) const noexcept { return nm_.getIndexByName(index); } +bool NsSelectFuncInterface::getIndexByName(std::string_view name, int& index) const noexcept { return nm_.getIndexByName(name, index); } +int NsSelectFuncInterface::getIndexesCount() const noexcept { return nm_.indexes_.size(); } -const std::string& NsSelectFuncInterface::getIndexName(int id) const { return nm_.indexes_[id]->Name(); } -IndexType NsSelectFuncInterface::getIndexType(int id) const { return nm_.indexes_[id]->Type(); } -const FieldsSet& NsSelectFuncInterface::getIndexFields(int id) const { return nm_.indexes_[id]->Fields(); } -TagsPath NsSelectFuncInterface::getTagsPathForField(const std::string& jsonPath) const { return nm_.tagsMatcher_.path2tag(jsonPath); } +const std::string& NsSelectFuncInterface::getIndexName(int id) const noexcept { return nm_.indexes_[id]->Name(); } +IndexType NsSelectFuncInterface::getIndexType(int id) const noexcept { return nm_.indexes_[id]->Type(); } +const FieldsSet& NsSelectFuncInterface::getIndexFields(int id) const noexcept { return nm_.indexes_[id]->Fields(); } +TagsPath NsSelectFuncInterface::getTagsPathForField(std::string_view jsonPath) const noexcept { + return nm_.tagsMatcher_.path2tag(jsonPath); +} } // namespace reindexer diff --git a/cpp_src/core/selectfunc/nsselectfuncinterface.h b/cpp_src/core/selectfunc/nsselectfuncinterface.h index b20aa5079..7bab47f88 100644 --- a/cpp_src/core/selectfunc/nsselectfuncinterface.h +++ b/cpp_src/core/selectfunc/nsselectfuncinterface.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include namespace reindexer { class NamespaceImpl; @@ -10,15 +9,15 @@ class FieldsSet; class NsSelectFuncInterface { public: - NsSelectFuncInterface(const NamespaceImpl& nm) : nm_(nm) {} - const std::string& GetName() const; - int getIndexByName(const std::string& index) const; - bool getIndexByName(const std::string& name, int& index) const; - int getIndexesCount() const; - const std::string& getIndexName(int id) const; - IndexType getIndexType(int id) const; - const FieldsSet& getIndexFields(int id) const; - TagsPath getTagsPathForField(const std::string& jsonPath) const; + explicit NsSelectFuncInterface(const NamespaceImpl& nm) noexcept : nm_(nm) {} + const std::string& GetName() const noexcept; + int getIndexByName(std::string_view index) const noexcept; + bool getIndexByName(std::string_view name, int& index) const noexcept; + int getIndexesCount() const noexcept; + const std::string& getIndexName(int id) const noexcept; + IndexType getIndexType(int id) const noexcept; + const FieldsSet& getIndexFields(int id) const noexcept; + TagsPath getTagsPathForField(std::string_view jsonPath) const noexcept; private: const NamespaceImpl& nm_; diff --git a/cpp_src/core/selectfunc/selectfunc.cc b/cpp_src/core/selectfunc/selectfunc.cc index e9ed7417d..604571816 100644 --- a/cpp_src/core/selectfunc/selectfunc.cc +++ b/cpp_src/core/selectfunc/selectfunc.cc @@ -9,38 +9,36 @@ namespace reindexer { -static inline void ltrim(std::string &s) { +inline void ltrim(std::string &s) { s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { return !std::isspace(ch); })); } // trim from end (in place) -static inline void rtrim(std::string &s) { +inline void rtrim(std::string &s) { s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); } // trim from both ends (in place) -static inline void trim(std::string &s) { +inline void trim(std::string &s) { ltrim(s); rtrim(s); } -SelectFunction::Ptr SelectFunctionsHolder::AddNamespace(const Query &q, const NamespaceImpl &nm, bool force) { +SelectFunction::Ptr SelectFunctionsHolder::AddNamespace(const Query &q, const NamespaceImpl &nm, uint32_t nsid, bool force) { if (q.selectFunctions_.empty() && !force) { return nullptr; } else if (!q.selectFunctions_.empty()) { force_only_ = false; } - if (!querys_) { - querys_.reset(new fast_hash_map); + if (queries_.size() <= nsid) { + queries_.resize(nsid + 1); } - - NsSelectFuncInterface nm_interface(nm); - SelectFunction::Ptr func = std::make_shared(q, nm_interface); - return querys_->emplace(nm_interface.GetName(), func).first->second; + queries_[nsid] = std::make_shared(q, NsSelectFuncInterface(nm)); + return queries_[nsid]; } -SelectFunction::SelectFunction(const Query &q, NsSelectFuncInterface &nm) : nm_(nm), currCjsonFieldIdx_(nm.getIndexesCount()) { +SelectFunction::SelectFunction(const Query &q, NsSelectFuncInterface &&nm) : nm_(std::move(nm)), currCjsonFieldIdx_(nm.getIndexesCount()) { functions_.reserve(q.selectFunctions_.size()); for (auto &func : q.selectFunctions_) { SelectFuncParser parser; @@ -145,7 +143,7 @@ BaseFunctionCtx::Ptr SelectFunction::CreateCtx(int indexNo) { } BaseFunctionCtx::Ptr ctx; - IndexType indexType = nm_.getIndexType(indexNo); + const IndexType indexType = nm_.getIndexType(indexNo); if (IsComposite(indexType)) { int fieldNo = 0; @@ -163,22 +161,33 @@ BaseFunctionCtx::Ptr SelectFunction::CreateCtx(int indexNo) { auto it = functions_.find(indexNo); if (it != functions_.end()) { it->second.fieldNo = 0; - ctx = createCtx(it->second, ctx, nm_.getIndexType(indexNo)); + ctx = createCtx(it->second, ctx, indexType); } } - if (!ctx && IsFullText(nm_.getIndexType(indexNo))) { + if (!ctx && IsFullText(indexType)) { return createFuncForProc(indexNo); } return ctx; } void SelectFunctionsHolder::Process(QueryResults &res) { - if (!querys_ || querys_->empty() || force_only_) return; + if (queries_.empty() || force_only_) return; + bool hasFuncs = false; + for (auto &q : queries_) { + if (q) { + hasFuncs = true; + break; + } + } + if (!hasFuncs) return; + bool changed = false; - for (size_t i = 0; i < res.Count(); ++i) { - auto &pl_type = res.getPayloadType(res.Items()[i].Nsid()); - auto it = querys_->find(pl_type.Name()); - if (it != querys_->end()) { - if (it->second->ProcessItem(res.Items()[i], pl_type, res.stringsHolder_)) changed = true; + for (auto &item : res.Items()) { + const auto nsid = item.Nsid(); + if (queries_.size() <= nsid) { + continue; + } + if (auto &funcPtr = queries_[nsid]; funcPtr && funcPtr->ProcessItem(item, res.getPayloadType(nsid), res.stringsHolder_)) { + changed = true; } } res.nonCacheableData = changed; @@ -186,8 +195,8 @@ void SelectFunctionsHolder::Process(QueryResults &res) { bool SelectFunction::ProcessItem(ItemRef &res, PayloadType &pl_type, std::vector &stringsHolder) { bool changed = false; for (auto &func : functions_) { - if (!func.second.ctx) continue; - if (std::visit([&](auto &f) -> bool { return f.Process(res, pl_type, func.second, stringsHolder); }, func.second.func)) { + if (func.second.ctx && + std::visit([&](auto &f) -> bool { return f.Process(res, pl_type, func.second, stringsHolder); }, func.second.func)) { changed = true; } } diff --git a/cpp_src/core/selectfunc/selectfunc.h b/cpp_src/core/selectfunc/selectfunc.h index 842f1a3dd..6b5686589 100644 --- a/cpp_src/core/selectfunc/selectfunc.h +++ b/cpp_src/core/selectfunc/selectfunc.h @@ -12,7 +12,7 @@ class NamespaceImpl; class SelectFunction { public: typedef std::shared_ptr Ptr; - SelectFunction(const Query& q, NsSelectFuncInterface& nm); + SelectFunction(const Query& q, NsSelectFuncInterface&& nm); /// Processes selected item to apply sql function. /// @param res - ItemRef containing payload value. @@ -48,18 +48,21 @@ class SelectFunctionsHolder { /// Creates SelectFunction object for a query. /// @param q - query that contains sql function. /// @param nm - namespace of the query to be executed. + /// @param nsid - unique ns ID in the query. /// @param force - forces to create SelectFunction object /// even if the list of sql-functions in this query is empty. /// @return pointer to SelectFunction object or nullptr in case of error. - SelectFunction::Ptr AddNamespace(const Query& q, const NamespaceImpl& nm, bool force); + SelectFunction::Ptr AddNamespace(const Query& q, const NamespaceImpl& nm, uint32_t nsid, bool force); /// Processing of results of an executed query. /// @param res - results of query execution. void Process(QueryResults& res); private: + using MapT = h_vector; + /// Indicates if object is empty and was created wuth flag force = true. bool force_only_ = true; /// Container of sql functions for every namespace. - std::unique_ptr> querys_; + MapT queries_; }; } // namespace reindexer diff --git a/cpp_src/core/storage/leveldblogger.cc b/cpp_src/core/storage/leveldblogger.cc new file mode 100644 index 000000000..eba6dc16c --- /dev/null +++ b/cpp_src/core/storage/leveldblogger.cc @@ -0,0 +1,22 @@ +#include "leveldblogger.h" + +#include +#include + +// Using separate cc-file to be able to compile it with different options. +// Static LevelDB v1.23 is built with -fno-rtti by default and to inherit NoOpLogger from leveldb's logger, this file must be built with +// -fno-rtti to + +namespace reindexer { +namespace datastorage { + +class NoOpLogger : public leveldb::Logger { + void Logv(const char* /*format*/, va_list /*ap*/) override final {} +}; + +static NoOpLogger dummyLevelDBLogger; + +void SetDummyLogger(leveldb::Options& options) { options.info_log = &dummyLevelDBLogger; } + +} // namespace datastorage +} // namespace reindexer diff --git a/cpp_src/core/storage/leveldblogger.h b/cpp_src/core/storage/leveldblogger.h new file mode 100644 index 000000000..cd14a5242 --- /dev/null +++ b/cpp_src/core/storage/leveldblogger.h @@ -0,0 +1,13 @@ +#pragma once + +namespace leveldb { +class Options; +} + +namespace reindexer { +namespace datastorage { + +void SetDummyLogger(leveldb::Options& options); + +} +} // namespace reindexer diff --git a/cpp_src/core/storage/leveldbstorage.cc b/cpp_src/core/storage/leveldbstorage.cc index 00520a2fb..026a443d1 100644 --- a/cpp_src/core/storage/leveldbstorage.cc +++ b/cpp_src/core/storage/leveldbstorage.cc @@ -5,7 +5,9 @@ #include #include #include +#include "leveldblogger.h" #include "tools/assertrx.h" +#include "tools/fsops.h" namespace reindexer { namespace datastorage { @@ -125,6 +127,7 @@ Error LevelDbStorage::doOpen(const std::string& path, const StorageOpts& opts) { leveldb::Options options; options.create_if_missing = opts.IsCreateIfMissing(); options.max_open_files = 50; + SetDummyLogger(options); leveldb::DB* db; leveldb::Status status = leveldb::DB::Open(options, path, &db); @@ -144,7 +147,11 @@ void LevelDbStorage::doDestroy(const std::string& path) { db_.reset(); leveldb::Status status = leveldb::DestroyDB(path.c_str(), options); if (!status.ok()) { - printf("Cannot destroy DB: %s, %s\n", path.c_str(), status.ToString().c_str()); + fprintf(stderr, "Cannot destroy LevelDB's storage: %s, %s. Trying to remove files by the backup mechanism...\n", path.c_str(), + status.ToString().c_str()); + if (fs::RmDirAll(path) != 0) { + fprintf(stderr, "Unable to remove LevelDB's storage: %s, %s", path.c_str(), strerror(errno)); + } } } diff --git a/cpp_src/core/storage/leveldbstorage.h b/cpp_src/core/storage/leveldbstorage.h index 291396452..472c1d696 100644 --- a/cpp_src/core/storage/leveldbstorage.h +++ b/cpp_src/core/storage/leveldbstorage.h @@ -2,6 +2,7 @@ #ifdef REINDEX_WITH_LEVELDB +#include #include #include #include "basestorage.h" diff --git a/cpp_src/core/storage/rocksdbstorage.cc b/cpp_src/core/storage/rocksdbstorage.cc index 4d20d8ef7..9535275b9 100644 --- a/cpp_src/core/storage/rocksdbstorage.cc +++ b/cpp_src/core/storage/rocksdbstorage.cc @@ -6,6 +6,7 @@ #include #include #include +#include "tools/fsops.h" namespace reindexer { namespace datastorage { @@ -143,8 +144,10 @@ void RocksDbStorage::doDestroy(const std::string& path) { options.create_if_missing = true; db_.reset(); rocksdb::Status status = rocksdb::DestroyDB(path.c_str(), options); - if (!status.ok()) { - printf("Cannot destroy DB: %s, %s\n", path.c_str(), status.ToString().c_str()); + fprintf(stderr, "Cannot destroy RocksDB's storage: %s, %s. Trying to remove files by the backup mechanism...\n", path.c_str(), + status.ToString().c_str()); + if (fs::RmDirAll(path) != 0) { + fprintf(stderr, "Unable to remove RocksDB's storage: %s, %s", path.c_str(), strerror(errno)); } } diff --git a/cpp_src/core/transaction.cc b/cpp_src/core/transaction.cc index 6e54bc431..fc39bad7f 100644 --- a/cpp_src/core/transaction.cc +++ b/cpp_src/core/transaction.cc @@ -67,4 +67,8 @@ Transaction::time_point Transaction::GetStartTime() const { return impl_->startTime_; } +void Transaction::ValidatePK(const FieldsSet &pkFields) { + if (impl_) impl_->ValidatePK(pkFields); +} + } // namespace reindexer diff --git a/cpp_src/core/transaction.h b/cpp_src/core/transaction.h index bd68418b4..0ab5c8b43 100644 --- a/cpp_src/core/transaction.h +++ b/cpp_src/core/transaction.h @@ -42,6 +42,7 @@ class Transaction { const std::vector &GetSteps() const; bool IsTagsUpdated() const; time_point GetStartTime() const; + void ValidatePK(const FieldsSet &pkFields); protected: std::unique_ptr impl_; diff --git a/cpp_src/core/transactionimpl.cc b/cpp_src/core/transactionimpl.cc index e9f47393c..a7ccb4f16 100644 --- a/cpp_src/core/transactionimpl.cc +++ b/cpp_src/core/transactionimpl.cc @@ -21,15 +21,14 @@ Item TransactionImpl::GetItem(TransactionStep &&st) { return Item(new ItemImpl(payloadType_, tagsMatcher_, pkFields_, schema_, std::move(st.itemData_))); } -TransactionImpl::TransactionImpl(const std::string &nsName, const PayloadType &pt, const TagsMatcher &tm, const FieldsSet &pf, - std::shared_ptr schema) - : payloadType_(pt), - tagsMatcher_(tm), - pkFields_(pf), - schema_(std::move(schema)), - nsName_(nsName), - tagsUpdated_(false), - startTime_(std::chrono::high_resolution_clock::now()) {} +void TransactionImpl::ValidatePK(const FieldsSet &pkFields) { + std::lock_guard lck(mtx_); + if (hasDeleteItemSteps_ && rx_unlikely(pkFields != pkFields_)) { + throw Error( + errNotValid, + "Transaction has Delete-calls and it's PK metadata is outdated (probably PK has been change during the transaction creation)"); + } +} void TransactionImpl::UpdateTagsMatcherFromItem(ItemImpl *ritem) { if (ritem->Type().get() != payloadType_.get() || (ritem->tagsMatcher().isUpdated() && !tagsMatcher_.try_merge(ritem->tagsMatcher()))) { @@ -76,6 +75,7 @@ void TransactionImpl::Delete(Item &&item) { void TransactionImpl::Modify(Item &&item, ItemModifyMode mode) { std::unique_lock lock(mtx_); checkTagsMatcher(item); + hasDeleteItemSteps_ = hasDeleteItemSteps_ || (mode == ModeDelete); steps_.emplace_back(TransactionStep{std::move(item), mode}); } diff --git a/cpp_src/core/transactionimpl.h b/cpp_src/core/transactionimpl.h index afe7641f7..89baa10d9 100644 --- a/cpp_src/core/transactionimpl.h +++ b/cpp_src/core/transactionimpl.h @@ -26,7 +26,15 @@ class TransactionStep { class TransactionImpl { public: TransactionImpl(const std::string &nsName, const PayloadType &pt, const TagsMatcher &tm, const FieldsSet &pf, - std::shared_ptr schema); + std::shared_ptr schema) + : payloadType_(pt), + tagsMatcher_(tm), + pkFields_(pf), + schema_(std::move(schema)), + nsName_(nsName), + tagsUpdated_(false), + hasDeleteItemSteps_(false), + startTime_(std::chrono::high_resolution_clock::now()) {} void Insert(Item &&item); void Update(Item &&item); @@ -38,6 +46,7 @@ class TransactionImpl { void UpdateTagsMatcherFromItem(ItemImpl *ritem); Item NewItem(); Item GetItem(TransactionStep &&st); + void ValidatePK(const FieldsSet &pkFields); const std::string &GetName() { return nsName_; } @@ -51,6 +60,7 @@ class TransactionImpl { std::vector steps_; std::string nsName_; bool tagsUpdated_; + bool hasDeleteItemSteps_; std::mutex mtx_; Transaction::time_point startTime_; }; diff --git a/cpp_src/core/txstats.h b/cpp_src/core/txstats.h index f30d7e27a..30e364909 100644 --- a/cpp_src/core/txstats.h +++ b/cpp_src/core/txstats.h @@ -1,9 +1,9 @@ #pragma once #include "core/transaction.h" +#include "core/transactionimpl.h" #include "namespace/namespacestat.h" #include "perfstatcounter.h" -#include "tools/stringstools.h" namespace reindexer { diff --git a/cpp_src/core/type_consts_helpers.cc b/cpp_src/core/type_consts_helpers.cc index 18fd15e1c..60db10409 100644 --- a/cpp_src/core/type_consts_helpers.cc +++ b/cpp_src/core/type_consts_helpers.cc @@ -110,6 +110,8 @@ namespace reindexer { return "unknown"sv; } +} // namespace reindexer + [[nodiscard]] std::string_view JoinTypeName(JoinType type) { using namespace std::string_view_literals; @@ -126,5 +128,3 @@ namespace reindexer { assertrx(false); return "unknown"sv; } - -} // namespace reindexer diff --git a/cpp_src/core/type_consts_helpers.h b/cpp_src/core/type_consts_helpers.h index 2fc5e36df..d4d32897e 100644 --- a/cpp_src/core/type_consts_helpers.h +++ b/cpp_src/core/type_consts_helpers.h @@ -11,6 +11,18 @@ namespace reindexer { [[nodiscard]] std::string_view TagTypeToStr(TagType); [[nodiscard]] std::string_view AggTypeToStr(AggType t) noexcept; +constexpr bool IsComposite(IndexType type) noexcept { + return type == IndexCompositeBTree || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT || type == IndexCompositeHash; +} + +constexpr bool IsFullText(IndexType type) noexcept { + return type == IndexFastFT || type == IndexFuzzyFT || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT; +} + +constexpr bool IsFastFullText(IndexType type) noexcept { return type == IndexFastFT || type == IndexCompositeFastFT; } + +} // namespace reindexer + /// Get readable Join Type /// @param type - join type /// @returns string with join type name @@ -132,15 +144,3 @@ T& operator<<(T& os, CollateMode m) { } std::abort(); } - -constexpr bool IsComposite(IndexType type) noexcept { - return type == IndexCompositeBTree || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT || type == IndexCompositeHash; -} - -constexpr bool IsFullText(IndexType type) noexcept { - return type == IndexFastFT || type == IndexFuzzyFT || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT; -} - -constexpr bool IsFastFullText(IndexType type) noexcept { return type == IndexFastFT || type == IndexCompositeFastFT; } - -} // namespace reindexer diff --git a/cpp_src/estl/chunk.h b/cpp_src/estl/chunk.h index e66e7dabb..1174b60af 100644 --- a/cpp_src/estl/chunk.h +++ b/cpp_src/estl/chunk.h @@ -7,7 +7,8 @@ namespace reindexer { class chunk { public: - chunk() : data_(nullptr), len_(0), offset_(0), cap_(0) {} + chunk() noexcept : data_(nullptr), len_(0), offset_(0), cap_(0) {} + chunk(uint8_t *data, size_t len, size_t cap, size_t offset = 0) noexcept : data_(data), len_(len), offset_(offset), cap_(cap) {} ~chunk() { delete[] data_; } chunk(const chunk &) = delete; chunk &operator=(const chunk &) = delete; @@ -35,21 +36,56 @@ class chunk { } return *this; } - void append(std::string_view data) { + void append(std::string_view data) { append_impl(data, std::max(size_t(0x1000), size_t(len_ + data.size()))); } + void append_strict(std::string_view data) { append_impl(data, len_ + data.size()); } + + size_t size() const noexcept { return len_ - offset_; } + uint8_t *data() const noexcept { return data_ + offset_; } + size_t capacity() const noexcept { return cap_; } + size_t len() const noexcept { return len_; } + size_t offset() const noexcept { return offset_; } + + void clear() noexcept { + len_ = 0; + offset_ = 0; + } + void shift(size_t offset) noexcept { offset_ += offset; } + uint8_t *release() noexcept { + auto res = data_; + data_ = nullptr; + return res; + } + + void shrink(size_t k) { + if (k * size() >= cap_) { + return; + } + + cap_ = k * size(); + uint8_t *newdata = new uint8_t[cap_]; + if (data_) { + memcpy(newdata, data(), size()); + len_ = size(); + offset_ = 0; + } + delete[] data_; + data_ = newdata; + } + +private: + void append_impl(std::string_view data, size_t newCapacity) { if (!data_ || len_ + data.size() > cap_) { - cap_ = std::max(size_t(0x1000), size_t(len_ + data.size())); + cap_ = newCapacity; uint8_t *newdata = new uint8_t[cap_]; if (data_) { memcpy(newdata, data_, len_); } - delete data_; + delete[] data_; data_ = newdata; } memcpy(data_ + len_, data.data(), data.size()); len_ += data.size(); } - size_t size() { return len_ - offset_; } - uint8_t *data() { return data_ + offset_; } uint8_t *data_; size_t len_; diff --git a/cpp_src/estl/chunk_buf.h b/cpp_src/estl/chunk_buf.h index 3ee04cc05..daed5c466 100644 --- a/cpp_src/estl/chunk_buf.h +++ b/cpp_src/estl/chunk_buf.h @@ -45,13 +45,12 @@ class chain_buf { assertrx(head_ != tail_); chunk &cur = ring_[tail_]; if (cur.size() > nread) { - cur.offset_ += nread; + cur.shift(nread); break; } nread -= cur.size(); - cur.len_ = 0; - cur.offset_ = 0; - if (free_.size() < ring_.size() && cur.cap_ < 0x10000) + cur.clear(); + if (free_.size() < ring_.size() && cur.capacity() < 0x10000) free_.push_back(std::move(cur)); else cur = chunk(); diff --git a/cpp_src/gtests/bench/fixtures/aggregation.cc b/cpp_src/gtests/bench/fixtures/aggregation.cc index 3641684f3..6523df608 100644 --- a/cpp_src/gtests/bench/fixtures/aggregation.cc +++ b/cpp_src/gtests/bench/fixtures/aggregation.cc @@ -1,6 +1,6 @@ #include "aggregation.h" #include "core/cjson/jsonbuilder.h" -#include "tools/random.h" +#include "tools/randompoint.h" template void Aggregation::Insert(State& state) { diff --git a/cpp_src/gtests/bench/fixtures/api_tv_composite.cc b/cpp_src/gtests/bench/fixtures/api_tv_composite.cc index fdeceac96..dc131e080 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_composite.cc +++ b/cpp_src/gtests/bench/fixtures/api_tv_composite.cc @@ -299,13 +299,14 @@ void ApiTvComposite::RangeHashCompositeIntStr(benchmark::State& state) { } void ApiTvComposite::RangeTreeIntSortByHashInt(State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {idRange.first, idRange.second}).Sort("age", false).Limit(20); + q.Where("id", CondRange, {idRange.first, idRange.second}).Sort("age"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -314,13 +315,14 @@ void ApiTvComposite::RangeTreeIntSortByHashInt(State& state) { } void ApiTvComposite::RangeTreeIntSortByTreeInt(State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {idRange.first, idRange.second}).Sort("year", false).Limit(20); + q.Where("id"sv, CondRange, {idRange.first, idRange.second}).Sort("year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -329,13 +331,14 @@ void ApiTvComposite::RangeTreeIntSortByTreeInt(State& state) { } void ApiTvComposite::RangeTreeStrSortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("age", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("age"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -344,13 +347,14 @@ void ApiTvComposite::RangeTreeStrSortByHashInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeStrSortByTreeInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("year", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -359,6 +363,7 @@ void ApiTvComposite::RangeTreeStrSortByTreeInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeDoubleSortByTreeInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); @@ -366,7 +371,7 @@ void ApiTvComposite::RangeTreeDoubleSortByTreeInt(benchmark::State& state) { auto leftRate = random(0.0, 4.99); auto rightRate = random(5.0, 10.0); - q.Where("rate", CondRange, {leftRate, rightRate}).Sort("year", false).Limit(20); + q.Where("rate"sv, CondRange, {leftRate, rightRate}).Sort("year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -375,6 +380,7 @@ void ApiTvComposite::RangeTreeDoubleSortByTreeInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeDoubleSortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); @@ -382,7 +388,7 @@ void ApiTvComposite::RangeTreeDoubleSortByHashInt(benchmark::State& state) { auto leftRate = random(0.0, 4.99); auto rightRate = random(5.0, 10.0); - q.Where("rate", CondRange, {leftRate, rightRate}).Sort("age", false).Limit(20); + q.Where("rate"sv, CondRange, {leftRate, rightRate}).Sort("age"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -391,13 +397,14 @@ void ApiTvComposite::RangeTreeDoubleSortByHashInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeStrSortByHashStrCollateASCII(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("location", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("location"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -406,13 +413,14 @@ void ApiTvComposite::RangeTreeStrSortByHashStrCollateASCII(benchmark::State& sta } void ApiTvComposite::RangeTreeStrSortByHashStrCollateUTF8(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("name", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("name"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -421,11 +429,12 @@ void ApiTvComposite::RangeTreeStrSortByHashStrCollateUTF8(benchmark::State& stat } void ApiTvComposite::SortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id", false).Limit(20); + q.Sort("id"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -434,11 +443,12 @@ void ApiTvComposite::SortByHashInt(benchmark::State& state) { } void ApiTvComposite::ForcedSortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id", false, {10, 20, 30, 40, 50}).Limit(20); + q.Sort("id"sv, false, {10, 20, 30, 40, 50}).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -447,11 +457,12 @@ void ApiTvComposite::ForcedSortByHashInt(benchmark::State& state) { } void ApiTvComposite::ForcedSortWithSecondCondition(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id", false, {10, 20, 30, 40, 50}).Sort("location", false).Limit(20); + q.Sort("id"sv, false, {10, 20, 30, 40, 50}).Sort("location"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -474,11 +485,12 @@ void ApiTvComposite::Query2CondIdSetComposite(benchmark::State& state) { } void ApiTvComposite::SortByHashStrCollateASCII(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("location", false).Limit(20); + q.Sort("location"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -487,11 +499,12 @@ void ApiTvComposite::SortByHashStrCollateASCII(benchmark::State& state) { } void ApiTvComposite::SortByHashStrCollateUTF8(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("name", false).Limit(20); + q.Sort("name"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -500,11 +513,12 @@ void ApiTvComposite::SortByHashStrCollateUTF8(benchmark::State& state) { } void ApiTvComposite::SortByHashCompositeIntInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+start_time", false).Limit(20); + q.Sort("id+start_time"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -513,11 +527,12 @@ void ApiTvComposite::SortByHashCompositeIntInt(benchmark::State& state) { } void ApiTvComposite::SortByHashCompositeIntStr(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+genre", false).Limit(20); + q.Sort("id+genre"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -526,11 +541,12 @@ void ApiTvComposite::SortByHashCompositeIntStr(benchmark::State& state) { } void ApiTvComposite::SortByTreeCompositeIntInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+year", false).Limit(20); + q.Sort("id+year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -539,11 +555,12 @@ void ApiTvComposite::SortByTreeCompositeIntInt(benchmark::State& state) { } void ApiTvComposite::SortByTreeCompositeIntStrCollateUTF8(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+name", false).Limit(20); + q.Sort("id+name"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); diff --git a/cpp_src/gtests/bench/fixtures/api_tv_simple.cc b/cpp_src/gtests/bench/fixtures/api_tv_simple.cc index a8b2650a1..e628e2b5f 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_simple.cc +++ b/cpp_src/gtests/bench/fixtures/api_tv_simple.cc @@ -108,7 +108,9 @@ void ApiTvSimple::RegisterAllCases() { Register("Query2CondIdSet20000", &ApiTvSimple::Query2CondIdSet20000, this); #endif // !defined(REINDEX_WITH_ASAN) && !defined(REINDEX_WITH_TSAN) && !defined(RX_WITH_STDLIB_DEBUG) Register("FromCJSON", &ApiTvSimple::FromCJSON, this); + Register("FromCJSONPKOnly", &ApiTvSimple::FromCJSONPKOnly, this); Register("GetCJSON", &ApiTvSimple::GetCJSON, this); + Register("ExtractField", &ApiTvSimple::ExtractField, this); // NOLINTEND(*cplusplus.NewDeleteLeaks) } @@ -263,12 +265,13 @@ reindexer::Error ApiTvSimple::prepareCJsonBench() { auto err = db_->AddNamespace(cjsonNsDef); if (!err.ok()) return err; + fieldsToExtract_.clear(); itemForCjsonBench_ = std::make_unique(db_->NewItem(cjsonNsName_)); if (!itemForCjsonBench_->Status().ok()) return itemForCjsonBench_->Status(); wrSer_.Reset(); reindexer::JsonBuilder bld(wrSer_); constexpr size_t len = 10; - bld.Put("id", 0); + bld.Put("id", kCjsonBenchItemID); bld.Put("bool_-_index", rand() % 2); bld.Put("int_-_index", rand()); bld.Put("int_hash_index", rand()); @@ -296,20 +299,29 @@ reindexer::Error ApiTvSimple::prepareCJsonBench() { bld.Array("string_tree_array_index", randStringArray()); for (size_t i = 0; i < 10; ++i) { const std::string i_str = std::to_string(i); + fieldsToExtract_.emplace_back("bool_field_" + i_str); bld.Put("bool_field_" + i_str, rand() % 2); + fieldsToExtract_.emplace_back("int_field_" + i_str); bld.Put("int_field_" + i_str, rand()); + fieldsToExtract_.emplace_back("double_field_" + i_str); bld.Put("double_field_" + i_str, rand() / double(rand() + 1)); + fieldsToExtract_.emplace_back("string_field_" + i_str); bld.Put("string_field_" + i_str, randString(len)); bld.Array("bool_array_field_" + i_str, randBoolArray()); bld.Array("int_array_field_" + i_str, randIntArray()); bld.Array("double_array_field_" + i_str, randDoubleArray()); bld.Array("string_array_field_" + i_str, randStringArray()); { - auto obj = bld.Object("nested_obj_" + i_str); + const std::string nestedBase("nested_obj_" + i_str); + auto obj = bld.Object(nestedBase); obj.Put("bool_field", rand() % 2); + fieldsToExtract_.emplace_back(nestedBase + ".bool_field"); obj.Put("int_field", rand()); + fieldsToExtract_.emplace_back(nestedBase + ".int_field"); obj.Put("double_field", rand() / double(rand() + 1)); + fieldsToExtract_.emplace_back(nestedBase + ".double_field"); obj.Put("string_field", randString(len)); + fieldsToExtract_.emplace_back(nestedBase + ".string_field"); obj.Array("bool_array_field", randBoolArray()); obj.Array("int_array_field", randIntArray()); obj.Array("double_array_field", randDoubleArray()); @@ -453,6 +465,30 @@ void ApiTvSimple::FromCJSON(benchmark::State& state) { } } +void ApiTvSimple::FromCJSONPKOnly(benchmark::State& state) { + reindexer::Item item = db_->NewItem(cjsonNsName_); + { + AllocsTracker allocsTracker(state); + for (auto _ : state) { // NOLINT(*deadcode.DeadStores) + const auto err = item.FromCJSON(cjsonOfItem_, true); + if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + } + } + assertrx(item["id"].Get() == kCjsonBenchItemID); +} + +void ApiTvSimple::ExtractField(benchmark::State& state) { + assertrx(itemForCjsonBench_); + assertrx(fieldsToExtract_.size()); + AllocsTracker allocsTracker(state); + for (auto _ : state) { // NOLINT(*deadcode.DeadStores) + const auto& fieldName = fieldsToExtract_[rand() % fieldsToExtract_.size()]; + const auto va = VariantArray((*itemForCjsonBench_)[fieldName]); + if (va.size() != 1) state.SkipWithError(fmt::sprintf("Unexpected result size: %d", va.size()).c_str()); + } +} + void ApiTvSimple::StringsSelect(benchmark::State& state) { AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) diff --git a/cpp_src/gtests/bench/fixtures/api_tv_simple.h b/cpp_src/gtests/bench/fixtures/api_tv_simple.h index 7f1587d6b..45b9a0d4c 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_simple.h +++ b/cpp_src/gtests/bench/fixtures/api_tv_simple.h @@ -80,7 +80,9 @@ class ApiTvSimple : private BaseFixture { void Query4CondRangeTotal(State& state); void Query4CondRangeCachedTotal(State& state); void FromCJSON(State&); + void FromCJSONPKOnly(State&); void GetCJSON(State&); + void ExtractField(State&); void query2CondIdSet(State& state, const std::vector>& idsets); reindexer::Error prepareCJsonBench(); @@ -108,5 +110,7 @@ class ApiTvSimple : private BaseFixture { std::string innerJoinLowSelectivityRightNs_{"inner_join_low_selectivity_right_ns"}; std::string cjsonNsName_{"cjson_ns_name"}; std::unique_ptr itemForCjsonBench_; + std::vector fieldsToExtract_; + constexpr static int kCjsonBenchItemID = 9973; std::string cjsonOfItem_; }; diff --git a/cpp_src/gtests/bench/fixtures/ft_fixture.cc b/cpp_src/gtests/bench/fixtures/ft_fixture.cc index b02143761..955ee9d66 100644 --- a/cpp_src/gtests/bench/fixtures/ft_fixture.cc +++ b/cpp_src/gtests/bench/fixtures/ft_fixture.cc @@ -7,6 +7,7 @@ #include "core/cjson/jsonbuilder.h" #include "core/ft/config/ftfastconfig.h" +#include "tools/errors.h" #include "tools/stringstools.h" #include @@ -241,18 +242,33 @@ reindexer::Item FullText::MakeItem(benchmark::State&) { void FullText::BuildInsertSteps(State& state) { AllocsTracker allocsTracker(state, printFlags); - db_->DropNamespace(nsdef_.name); + auto err = db_->DropNamespace(nsdef_.name); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } id_seq_->Reset(); - auto err = BaseFixture::Initialize(); + err = BaseFixture::Initialize(); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } size_t i = 0; size_t mem = 0; + assert(!words_.empty()); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) auto item = MakeSpecialItem(); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + assertf(item.Status().ok(), "%s", item.Status().what()); + } err = db_->Insert(nsdef_.name, item); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } if (i % 12000 == 0) { Query q(nsdef_.name); @@ -279,7 +295,10 @@ void FullText::Insert(State& state) { AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) auto item = MakeItem(state); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + continue; + } auto err = db_->Insert(nsdef_.name, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -290,10 +309,11 @@ void FullText::Insert(State& state) { } void FullText::BuildCommonIndexes(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Where("year", CondRange, {2010, 2016}).Limit(20).Sort("year", false); + q.Where("year"sv, CondRange, {2010, 2016}).Limit(20).Sort("year"sv, false); std::this_thread::sleep_for(std::chrono::milliseconds(2000)); QueryResults qres; @@ -325,7 +345,10 @@ void FullText::BuildAndInsertLowWordsDiversityNs(State& state) { item["description1"] = d1; item["description2"] = d2; - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + continue; + } auto err = db_->Insert(lowWordsDiversityNsDef_.name, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -908,7 +931,13 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { ftCfg.optimization = opt; ftIndexOpts.config = ftCfg.GetJson({}); AllocsTracker allocsTracker(state, printFlags); - db_->DropNamespace(alternatingNs_); + auto err = db_->DropNamespace(alternatingNs_); + if (!err.ok()) { + if (err.code() != errNotFound || err.what() != "Namespace '" + alternatingNs_ + "' does not exist") { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } + } for (auto _ : state) { // NOLINT(*deadcode.DeadStores) NamespaceDef nsDef{alternatingNs_}; nsDef.AddIndex("id", "hash", "int", IndexOpts().PK()) @@ -918,7 +947,10 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { .AddIndex("search_comp", {"search1", "search2"}, "text", "composite", ftIndexOpts) .AddIndex("search_comp_not_index_fields", {"field1", "field2"}, "text", "composite", ftIndexOpts); auto err = db_->AddNamespace(nsDef); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } values_.clear(); values_.reserve(kNsSize); reindexer::WrSerializer ser; @@ -934,18 +966,25 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { bld.Put("rand", rand()); bld.End(); auto item = db_->NewItem(alternatingNs_); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + continue; + } err = item.FromJSON(ser.Slice()); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + continue; + } err = db_->Insert(alternatingNs_, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); } } - auto err = db_->Commit(alternatingNs_); + err = db_->Commit(alternatingNs_); if (!err.ok()) state.SkipWithError(err.what().c_str()); // Init index build + assert(!values_.empty()); Query q = Query(alternatingNs_) .Where("search1", CondEq, @@ -970,6 +1009,7 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { void FullText::updateAlternatingNs(reindexer::WrSerializer& ser, benchmark::State& state) { using namespace std::string_literals; + assert(!values_.empty()); const int i = randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(values_.size() - 1)}); ser.Reset(); reindexer::JsonBuilder bld(ser); @@ -982,9 +1022,15 @@ void FullText::updateAlternatingNs(reindexer::WrSerializer& ser, benchmark::Stat bld.End(); auto item = db_->NewItem(alternatingNs_); item.Unsafe(false); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + return; + } auto err = item.FromJSON(ser.Slice()); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + return; + } err = db_->Update(alternatingNs_, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -997,6 +1043,7 @@ void FullText::updateAlternatingNs(reindexer::WrSerializer& ser, benchmark::Stat } void FullText::AlternatingUpdatesAndSelects(benchmark::State& state) { + assert(!values_.empty()); reindexer::WrSerializer ser; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) @@ -1015,6 +1062,7 @@ void FullText::AlternatingUpdatesAndSelects(benchmark::State& state) { } void FullText::AlternatingUpdatesAndSelectsByComposite(benchmark::State& state) { + assert(!values_.empty()); reindexer::WrSerializer ser; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) @@ -1030,6 +1078,7 @@ void FullText::AlternatingUpdatesAndSelectsByComposite(benchmark::State& state) } void FullText::AlternatingUpdatesAndSelectsByCompositeByNotIndexFields(benchmark::State& state) { + assert(!values_.empty()); reindexer::WrSerializer ser; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) diff --git a/cpp_src/gtests/bench/fixtures/ft_fixture.h b/cpp_src/gtests/bench/fixtures/ft_fixture.h index 309c0de75..13f69eb43 100644 --- a/cpp_src/gtests/bench/fixtures/ft_fixture.h +++ b/cpp_src/gtests/bench/fixtures/ft_fixture.h @@ -200,7 +200,7 @@ class FullText : private BaseFixture { void updateAlternatingNs(reindexer::WrSerializer&, benchmark::State&); reindexer::Error readDictFile(const std::string& fileName, std::vector& words); - const char* alternatingNs_ = "FtAlternatingUpdatesAndSelects"; + const std::string alternatingNs_ = "FtAlternatingUpdatesAndSelects"; size_t raw_data_sz_ = 0; std::mt19937 randomEngine_{1}; diff --git a/cpp_src/gtests/bench/fixtures/geometry.cc b/cpp_src/gtests/bench/fixtures/geometry.cc index a82702658..fbcb9e52f 100644 --- a/cpp_src/gtests/bench/fixtures/geometry.cc +++ b/cpp_src/gtests/bench/fixtures/geometry.cc @@ -1,6 +1,6 @@ #include "geometry.h" #include "core/cjson/jsonbuilder.h" -#include "tools/random.h" +#include "tools/randompoint.h" namespace { @@ -30,7 +30,7 @@ void Geometry::GetDWithin(benchmark::State& state) { benchmark::AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) reindexer::Query q(nsdef_.name); - q.DWithin("point", randPoint(kRange), kRange / N); + q.DWithin("point", reindexer::randPoint(kRange), kRange / N); reindexer::QueryResults qres; auto err = db_->Select(q, qres); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -97,7 +97,7 @@ reindexer::Item Geometry::MakeItem(benchmark::State& state) { wrSer_.Reset(); reindexer::JsonBuilder bld(wrSer_); bld.Put("id", id_++); - const reindexer::Point point = randPoint(kRange); + const reindexer::Point point = reindexer::randPoint(kRange); double coords[]{point.X(), point.Y()}; bld.Array("point", reindexer::span(coords, 2)); bld.End(); diff --git a/cpp_src/gtests/bench/ft_bench.cc b/cpp_src/gtests/bench/ft_bench.cc index 2358cd617..05b81d1de 100644 --- a/cpp_src/gtests/bench/ft_bench.cc +++ b/cpp_src/gtests/bench/ft_bench.cc @@ -30,11 +30,12 @@ int main(int argc, char** argv) { } shared_ptr DB = std::make_shared(); - DB->Connect("builtin://" + kStoragePath); + auto err = DB->Connect("builtin://" + kStoragePath); + if (!err.ok()) return err.code(); FullText ft(DB.get(), "fulltext", kItemsInBenchDataset); - auto err = ft.Initialize(); + err = ft.Initialize(); if (!err.ok()) return err.code(); ::benchmark::Initialize(&argc, argv); diff --git a/cpp_src/gtests/bench/reindexer_bench.cc b/cpp_src/gtests/bench/reindexer_bench.cc index 2053b0d1f..216a8d42a 100644 --- a/cpp_src/gtests/bench/reindexer_bench.cc +++ b/cpp_src/gtests/bench/reindexer_bench.cc @@ -39,7 +39,8 @@ int main(int argc, char** argv) { } shared_ptr DB = std::make_shared(); - DB->Connect("builtin://" + kStoragePath); + auto err = DB->Connect("builtin://" + kStoragePath); + if (!err.ok()) return err.code(); JoinItems joinItems(DB.get(), 500); ApiTvSimple apiTvSimple(DB.get(), "ApiTvSimple", kItemsInBenchDataset); @@ -48,7 +49,7 @@ int main(int argc, char** argv) { Geometry geometry(DB.get(), "Geometry", kItemsInBenchDataset); Aggregation aggregation(DB.get(), "Aggregation", kItemsInBenchDataset); - auto err = apiTvSimple.Initialize(); + err = apiTvSimple.Initialize(); if (!err.ok()) return err.code(); err = apiTvSimpleComparators.Initialize(); diff --git a/cpp_src/gtests/tests/API/base_tests.cc b/cpp_src/gtests/tests/API/base_tests.cc index a466ceb28..9b8d30122 100644 --- a/cpp_src/gtests/tests/API/base_tests.cc +++ b/cpp_src/gtests/tests/API/base_tests.cc @@ -89,11 +89,14 @@ TEST_F(ReindexerApi, RenameNamespace) { auto getRowsInJSON = [&](const std::string& namespaceName, std::vector& resStrings) { QueryResults result; - rt.reindexer->Select(Query(namespaceName), result); + auto err = rt.reindexer->Select(Query(namespaceName), result); + ASSERT_TRUE(err.ok()) << err.what(); resStrings.clear(); for (auto it = result.begin(); it != result.end(); ++it) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer sr; - it.GetJSON(sr, false); + err = it.GetJSON(sr, false); + ASSERT_TRUE(err.ok()) << err.what(); std::string_view sv = sr.Slice(); resStrings.emplace_back(sv.data(), sv.size()); } @@ -246,8 +249,7 @@ TEST_F(ReindexerApi, DistinctCompositeIndex) { EXPECT_TRUE(err.ok()) << err.what(); } - Query q; - q._namespace = default_namespace; + Query q{default_namespace}; q.Distinct("v1+v2"); { QueryResults qr; @@ -1333,15 +1335,15 @@ TEST_F(ReindexerApi, DistinctQueriesEncodingTest) { std::string dsl = q1.GetJSON(); Query q2; q2.FromJSON(dsl); - EXPECT_TRUE(q1 == q2); + EXPECT_EQ(q1, q2) << "q1: " << q1.GetSQL() << "\nq2: " << q2.GetSQL(); Query q3{Query(default_namespace).Distinct("name").Distinct("city").Where("id", CondGt, static_cast(10))}; std::string sql2 = q3.GetSQL(); Query q4; q4.FromSQL(sql2); - EXPECT_TRUE(q3 == q4); - EXPECT_TRUE(sql2 == q4.GetSQL()); + EXPECT_EQ(q3, q4) << "q3: " << q3.GetSQL() << "\nq4: " << q4.GetSQL(); + EXPECT_EQ(sql2, q4.GetSQL()); } TEST_F(ReindexerApi, ContextCancelingTest) { @@ -1801,3 +1803,76 @@ TEST_F(ReindexerApi, InsertIncorrectItemWithJsonPathsDuplication) { ASSERT_EQ(qr.Count(), 0); } } + +TEST_F(ReindexerApi, IntFieldConvertToStringIndexTest) { + Error err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); + ASSERT_TRUE(err.ok()) << err.what(); + + err = rt.reindexer->AddIndex(default_namespace, {"id", "hash", "int", IndexOpts().PK()}); + ASSERT_TRUE(err.ok()) << err.what(); + + static int id = 0; + enum class Order { InsertThenAddIndex, AddIndexThenUpdate }; + + auto testImpl = [this](Order order) { + std::srand(std::time(0)); + int value = std::rand(); + auto indexName = fmt::sprintf("data_%d", id); + auto indexPaths = order == Order::AddIndexThenUpdate ? reindexer::JsonPaths{"n." + indexName} : reindexer::JsonPaths{indexName}; + auto insert = [this](const char* tmplt, auto&&... args) { + Item item(rt.reindexer->NewItem(default_namespace)); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + auto err = item.FromJSON(fmt::sprintf(tmplt, std::forward(args)...)); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->Insert(default_namespace, item); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + ASSERT_TRUE(err.ok()) << err.what(); + }; + + auto update = [&] { + QueryResults qr; + auto err = rt.reindexer->Select( + fmt::sprintf("UPDATE %s SET n = {\"%s\":%d} where id = %d", default_namespace, indexName, value, id), qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.Count(), 1); + }; + + auto addIndex = [&] { + auto err = rt.reindexer->AddIndex(default_namespace, {indexName, std::move(indexPaths), "hash", "string", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + }; + + auto checkResult = [&](const std::string& searchIndex, const std::string& searchValue) { + QueryResults qr; + auto err = rt.reindexer->Select(Query(default_namespace).Where(searchIndex, CondEq, searchValue), qr); + ASSERT_TRUE(err.ok()) << err.what(); + + ASSERT_EQ(qr.Count(), 1); + + auto item = qr.begin().GetItem(); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + ASSERT_TRUE(Variant(item[indexName]).Type().Is()) << Variant(item[indexName]).Type().Name(); + ASSERT_EQ(item[indexName].As(), std::to_string(value)); + }; + + switch (order) { + case Order::InsertThenAddIndex: { + insert("{\"id\":%d,\"%s\":%d})", id, indexName, value); + addIndex(); + break; + } + case Order::AddIndexThenUpdate: { + addIndex(); + insert("{\"id\":%d}", id); + update(); + break; + } + } + checkResult("id", std::to_string(id)); + checkResult(indexName, std::to_string(value)); + id++; + }; + + testImpl(Order::InsertThenAddIndex); + testImpl(Order::AddIndexThenUpdate); +} \ No newline at end of file diff --git a/cpp_src/gtests/tests/fixtures/ft_api.cc b/cpp_src/gtests/tests/fixtures/ft_api.cc index d4668d665..9f5c9ac81 100644 --- a/cpp_src/gtests/tests/fixtures/ft_api.cc +++ b/cpp_src/gtests/tests/fixtures/ft_api.cc @@ -61,7 +61,10 @@ reindexer::Error FTApi::SetFTConfig(const reindexer::FtFastConfig& ftCfg, const fieldsMap.emplace(fields[i], i); } std::vector nses; - rt.reindexer->EnumNamespaces(nses, reindexer::EnumNamespacesOpts().WithFilter(ns)); + auto err = rt.reindexer->EnumNamespaces(nses, reindexer::EnumNamespacesOpts().WithFilter(ns)); + if (!err.ok()) { + return err; + } const auto it = std::find_if(nses[0].indexes.begin(), nses[0].indexes.end(), [&index](const reindexer::IndexDef& idef) { return idef.name_ == index; }); it->opts_.SetConfig(ftCfg.GetJson(fieldsMap)); @@ -174,11 +177,11 @@ reindexer::QueryResults FTApi::SimpleSelect3(std::string word) { return res; } -void FTApi::Delete(int id) { +reindexer::Error FTApi::Delete(int id) { reindexer::Item item = rt.NewItem("nm1"); item["id"] = id; - this->rt.reindexer->Delete("nm1", item); + return this->rt.reindexer->Delete("nm1", item); } reindexer::QueryResults FTApi::SimpleCompositeSelect(std::string word) { diff --git a/cpp_src/gtests/tests/fixtures/ft_api.h b/cpp_src/gtests/tests/fixtures/ft_api.h index 5e56bf8b3..542630dec 100644 --- a/cpp_src/gtests/tests/fixtures/ft_api.h +++ b/cpp_src/gtests/tests/fixtures/ft_api.h @@ -32,7 +32,7 @@ class FTApi : public ::testing::TestWithParam& indexes) const { + const FieldType fldType = std::visit( + reindexer::overloaded{[](const Child& c) noexcept { return c.type; }, [](const Children&) noexcept { return FieldType::Struct; }}, + content_); IndexOpts opts; - const bool pk = rnd.PkIndex(isPk); + const bool pk = rnd.PkIndex(isPk_); opts.PK(pk); - opts.Array(rnd.RndArrayField(isArray)); - opts.Sparse(rnd.SparseIndex(pk)); + opts.Array(rnd.RndArrayField(isArray_) == IsArray::Yes); + opts.Sparse(rnd.RndSparseIndex(isSparse_)); opts.Dense(rnd.DenseIndex()); opts.RTreeType(static_cast(rnd.RndInt(IndexOpts::Linear, IndexOpts::RStar))); - FieldType fldType = std::visit( - reindexer::overloaded{[](const Child& c) noexcept { return c.type; }, [](const Children&) noexcept { return FieldType::Struct; }}, - content); std::string fieldType = rnd.IndexFieldType(fldType); - std::string indexType = rnd.RndIndexType(fldType, isPk); + std::string indexType{ToText(rnd.RndIndexType(type_))}; reindexer::JsonPaths jsonPaths; std::visit(reindexer::overloaded{[&](const Child& c) { jsonPaths.push_back(scheme.GetJsonPath(c.fieldPath)); }, [&](const Children& c) { + jsonPaths.reserve(c.size()); for (const auto& child : c) { + if (rnd.RndBool(0.5)) { + std::vector scalarIndexes; + scalarIndexes.reserve(indexes.size()); + for (size_t i = 0, s = indexes.size(); i < s; ++i) { + if (const auto* c = std::get_if(&indexes[i].content_); + c && c->fieldPath == child.fieldPath) { + scalarIndexes.push_back(i); + } + } + if (!scalarIndexes.empty()) { + jsonPaths.push_back(indexes[rnd.RndWhich(scalarIndexes)].name_); + continue; + } + } jsonPaths.push_back(scheme.GetJsonPath(child.fieldPath)); } }}, - content); - return {name, std::move(jsonPaths), std::move(indexType), std::move(fieldType), std::move(opts), rnd.ExpiredIndex()}; + content_); + return {name_, std::move(jsonPaths), std::move(indexType), std::move(fieldType), std::move(opts), rnd.ExpiredIndex()}; } void Index::Dump(std::ostream& os, const NsScheme& scheme, size_t offset) const { for (size_t i = 0; i < offset; ++i) os << " "; os << "{\n"; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "name: " << name << '\n'; + os << "name: " << name_ << '\n'; + for (size_t i = 0; i <= offset; ++i) os << " "; + os << "type: " << type_ << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "pk: " << (isPk ? "true" : "false") << '\n'; + os << "pk: " << std::boolalpha << isPk_ << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "array: " << (isArray ? "true" : "false") << '\n'; + os << "array: " << std::boolalpha << IsArray() << '\n'; + for (size_t i = 0; i <= offset; ++i) os << " "; + os << "sparse: " << std::boolalpha << (IsSparse() == IsSparse::Yes) << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; std::visit(reindexer::overloaded{[&](const Child& child) { + os << "composite: false\n"; + for (size_t i = 0; i <= offset; ++i) os << " "; os << "field: {\n"; for (size_t i = 0; i < offset + 2; ++i) os << " "; os << "type: " << child.type << '\n'; @@ -48,6 +72,8 @@ void Index::Dump(std::ostream& os, const NsScheme& scheme, size_t offset) const os << "}\n"; }, [&](const Children& children) { + os << "composite: true\n"; + for (size_t i = 0; i <= offset; ++i) os << " "; os << "fields: [\n"; for (const auto& c : children) { for (size_t i = 0; i < offset + 2; ++i) os << " "; @@ -62,7 +88,7 @@ void Index::Dump(std::ostream& os, const NsScheme& scheme, size_t offset) const for (size_t i = 0; i <= offset; ++i) os << " "; os << "]\n"; }}, - content); + content_); } } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/index.h b/cpp_src/gtests/tests/fixtures/fuzzing/index.h index 1daeb9266..80407211c 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/index.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/index.h @@ -1,7 +1,9 @@ #pragma once -#include "ns_scheme.h" -#include "random_generator.h" +#include +#include +#include +#include "types.h" namespace reindexer { struct IndexDef; @@ -9,20 +11,43 @@ struct IndexDef; namespace fuzzing { -struct Index { - reindexer::IndexDef IndexDef(RandomGenerator&, const NsScheme&) const; +class RandomGenerator; +class NsScheme; +class Index { +public: struct Child { FieldType type; FieldPath fieldPath; }; using Children = std::vector; - std::string name; - std::variant content; - bool isPk{false}; - bool isArray{false}; + Index(std::string name, IndexType type, IsArray isArray, IsSparse isSparse, Children content) noexcept + : name_{std::move(name)}, type_{type}, content_{std::move(content)}, isArray_{isArray}, isSparse_{isSparse} {} + Index(std::string name, IndexType type, IsArray isArray, IsSparse isSparse, Child content) noexcept + : name_{std::move(name)}, type_{type}, content_{std::move(content)}, isArray_{isArray}, isSparse_{isSparse} {} + + const std::string& Name() const& noexcept { return name_; } + const std::string& Name() const&& = delete; + IndexType Type() const noexcept { return type_; } + const auto& Content() const& noexcept { return content_; } + const auto& Content() const&& = delete; + bool IsPk() const noexcept { return isPk_; } + void SetPk() noexcept { isPk_ = true; } + bool IsArray() const noexcept { return isArray_ == IsArray::Yes; } + auto IsSparse() const noexcept { return isSparse_; } + + reindexer::IndexDef IndexDef(RandomGenerator&, const NsScheme&, const std::vector&) const; + void Dump(std::ostream&, const NsScheme&, size_t offset) const; + +private: + std::string name_; + IndexType type_; + std::variant content_; + bool isPk_{false}; + enum IsArray isArray_ { IsArray::No }; + enum IsSparse isSparse_ { IsSparse::No }; }; } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc b/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc index 93d5f16fc..5a520d4a0 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc @@ -1,5 +1,8 @@ #include "ns.h" #include +#include "estl/overloaded.h" +#include "index.h" +#include "tools/assertrx.h" struct FieldPathHash { size_t operator()(const fuzzing::FieldPath& fp) const noexcept { @@ -30,47 +33,74 @@ static bool availablePkFieldType(FieldType ft) { } } -Ns::Ns(std::string name, std::ostream& os, RandomGenerator::ErrFactorType errorFactor) - : name_{std::move(name)}, rndGen_{os, errorFactor}, scheme_{name_, rndGen_} { - std::unordered_set generatedNames; +static bool availablePkIndexType(IndexType it) { + switch (it) { + case IndexType::Store: + case IndexType::FastFT: + case IndexType::FuzzyFT: + case IndexType::RTree: + return false; + case IndexType::Hash: + case IndexType::Tree: + case IndexType::Ttl: + return true; + default: + assertrx(false); + std::abort(); + } +} + +Ns::Ns(std::string name, RandomGenerator::ErrFactorType errorFactor) + : name_{std::move(name)}, rndGen_{errorFactor}, scheme_{name_, rndGen_} { + std::unordered_set usedIndexNames; std::unordered_set usedPaths; constexpr static size_t kMaxTries = 10; const size_t idxCount = rndGen_.IndexesCount(); const bool withErr = rndGen_.RndErr(); indexes_.reserve(idxCount); + std::vector scalarIndexes; + scalarIndexes.reserve(idxCount); for (size_t i = 0; i < idxCount; ++i) { const bool uniqueName = rndGen_.UniqueName(); - if (rndGen_.CompositeIndex()) { - bool fail = false; - Index index{uniqueName ? rndGen_.IndexName(generatedNames) : std::string{}, Index::Children{}}; - auto& children = std::get(index.content); - const size_t size = rndGen_.CompositeIndexSize(); - children.reserve(size); - for (size_t i = 0; i < size; ++i) { - auto fldPath = rndGen_.RndScalarField(scheme_); - FieldType fldType; - if (scheme_.IsStruct(fldPath)) { - if (!rndGen_.RndErr()) { - fail = true; - break; - } - fldType = rndGen_.RndFieldType(); + if (rndGen_.CompositeIndex(scalarIndexes.size())) { + bool array = false; + bool containsUuid = false; + std::string name; + Index::Children children; + const auto fields = rndGen_.RndFieldsForCompositeIndex(scalarIndexes); + children.reserve(fields.size()); + for (size_t f : fields) { + Index::Child fieldData; + if (f < indexes_.size()) { + const auto& idx = indexes_[f]; + fieldData = std::get(idx.Content()); + array |= idx.IsArray(); } else { - fldType = scheme_.GetFieldType(fldPath); - } - if (!uniqueName) { - if (!index.name.empty()) index.name += '+'; - if (fldPath.empty()) { - index.name += rndGen_.FieldName(generatedNames); + fieldData.fieldPath = rndGen_.RndScalarField(scheme_); + if (scheme_.IsStruct(fieldData.fieldPath)) { + fieldData.type = rndGen_.RndFieldType(); } else { - index.name += scheme_.GetJsonPath(fldPath); + fieldData.type = scheme_.GetFieldType(fieldData.fieldPath); } } - children.emplace_back(Index::Child{fldType, std::move(fldPath)}); + if (!uniqueName) { + if (!name.empty()) name += '+'; + name += scheme_.GetJsonPath(fieldData.fieldPath); + } + containsUuid |= fieldData.type == FieldType::Uuid; + children.emplace_back(std::move(fieldData)); + } + const auto indexType = + containsUuid ? rndGen_.RndIndexType({FieldType::Struct, FieldType::Uuid}) : rndGen_.RndIndexType({FieldType::Struct}); + if (uniqueName) { + name = rndGen_.IndexName(usedIndexNames); + } else if (!usedIndexNames.insert(name).second) { + name = rndGen_.IndexName(usedIndexNames); + usedIndexNames.insert(name); } - if (fail) continue; - index.isArray = rndGen_.RndArrayField(false); - indexes_.emplace_back(std::move(index)); + + indexes_.emplace_back(std::move(name), indexType, rndGen_.RndArrayField(array ? IsArray::Yes : IsArray::No), IsSparse::No, + std::move(children)); } else { FieldPath fldPath; size_t tryCounts = 0; @@ -82,47 +112,73 @@ Ns::Ns(std::string name, std::ostream& os, RandomGenerator::ErrFactorType errorF if (scheme_.IsStruct(fldPath)) { if (!rndGen_.RndErr()) continue; const auto fldType = rndGen_.RndFieldType(); - indexes_.emplace_back(Index{rndGen_.IndexName(generatedNames), Index::Child{fldType, std::move(fldPath)}}); + indexes_.emplace_back(rndGen_.IndexName(usedIndexNames), rndGen_.RndIndexType({fldType}), + rndGen_.RndBool(0.5) ? IsArray::Yes : IsArray::No, + rndGen_.RndBool(0.5) ? IsSparse::Yes : IsSparse::No, Index::Child{fldType, std::move(fldPath)}); } else { const auto fldType = scheme_.GetFieldType(fldPath); - const bool isArray = scheme_.IsArray(fldPath); - std::string idxName = uniqueName ? rndGen_.IndexName(generatedNames) : scheme_.GetJsonPath(fldPath); - indexes_.emplace_back(Index{std::move(idxName), Index::Child{fldType, std::move(fldPath)}}); - indexes_.back().isArray = rndGen_.RndArrayField(isArray); + const auto isArray = scheme_.IsArray(fldPath); + std::string idxName; + if (uniqueName) { + idxName = rndGen_.IndexName(usedIndexNames); + } else { + idxName = scheme_.GetJsonPath(fldPath); + if (!usedIndexNames.insert(idxName).second) { + idxName = rndGen_.IndexName(usedIndexNames); + usedIndexNames.insert(idxName); + } + } + indexes_.emplace_back(std::move(idxName), rndGen_.RndIndexType({fldType}), rndGen_.RndArrayField(isArray), + rndGen_.RndSparseIndex(fldType), Index::Child{fldType, std::move(fldPath)}); + } + if (const auto& idx = indexes_.back(); + !idx.IsArray() && idx.IsSparse() == IsSparse::No && + std::get(idx.Content()).type != FieldType::Point) { // TODO remove point check after #1352 + scalarIndexes.push_back(indexes_.size() - 1); } } } + if (rndGen_.RndErr()) { + // Do not set PK index + return; + } std::vector ii; for (size_t i = 0, s = indexes_.size(); i < s; ++i) { const auto& idx = indexes_[i]; - if (!idx.isArray && - (std::holds_alternative(idx.content) || availablePkFieldType(std::get(idx.content).type))) { + if (!idx.IsArray() && idx.IsSparse() == IsSparse::No && availablePkIndexType(idx.Type()) && + (std::holds_alternative(idx.Content()) || availablePkFieldType(std::get(idx.Content()).type))) { ii.push_back(i); } } if (ii.empty()) { - if (!rndGen_.RndErr()) { - auto path = scheme_.AddRndPkField(rndGen_); - const auto fldType = scheme_.GetFieldType(path); - std::string name = rndGen_.UniqueName() ? rndGen_.IndexName(generatedNames) : scheme_.GetJsonPath(path); - indexes_.emplace_back(Index{std::move(name), Index::Child{fldType, std::move(path)}}); - indexes_.back().isArray = false; - indexes_.back().isPk = true; + auto path = scheme_.AddRndPkField(rndGen_); + const auto fldType = scheme_.GetFieldType(path); + std::string name; + if (rndGen_.UniqueName()) { + name = rndGen_.IndexName(usedIndexNames); + } else { + name = scheme_.GetJsonPath(path); + if (!usedIndexNames.insert(name).second) { + name = rndGen_.IndexName(usedIndexNames); + usedIndexNames.insert(name); + } } + indexes_.emplace_back(std::move(name), rndGen_.RndPkIndexType({fldType}), IsArray::No, IsSparse::No, + Index::Child{fldType, std::move(path)}); + indexes_.back().SetPk(); } else { - indexes_[rndGen_.RndWhich(ii)].isPk = true; + indexes_[rndGen_.RndWhich(ii)].SetPk(); } } -void Ns::AddIndex(Index& index, bool isSparse) { - if (isSparse) return; - std::visit(reindexer::overloaded{[&](const Index::Child& c) { scheme_.AddIndex(c.fieldPath, isSparse); }, +void Ns::AddIndexToScheme(const Index& index, size_t indexNumber) { + std::visit(reindexer::overloaded{[&](const Index::Child& c) { scheme_.AddIndex(c.fieldPath, indexNumber, index.IsSparse()); }, [&](const Index::Children& c) { for (const auto& child : c) { - scheme_.AddIndex(child.fieldPath, isSparse); + scheme_.AddIndex(child.fieldPath, indexNumber, index.IsSparse()); } }}, - index.content); + index.Content()); } void Ns::Dump(std::ostream& os) const { diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns.h b/cpp_src/gtests/tests/fixtures/fuzzing/ns.h index 53baf5a3e..6c93abb46 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns.h @@ -1,21 +1,23 @@ #pragma once -#include "index.h" #include "ns_scheme.h" #include "random_generator.h" namespace fuzzing { +class Index; + class Ns { public: - Ns(std::string name, std::ostream&, RandomGenerator::ErrFactorType errorFactor); - std::vector& GetIndexes() noexcept { return indexes_; } + Ns(std::string name, RandomGenerator::ErrFactorType errorFactor); + const std::vector& GetIndexes() const& noexcept { return indexes_; } + std::vector& GetIndexes() & noexcept { return indexes_; } + const std::vector& GetIndexes() const&& = delete; const std::string& GetName() const noexcept { return name_; } const NsScheme& GetScheme() const noexcept { return scheme_; } RandomGenerator& GetRandomGenerator() noexcept { return rndGen_; } - void AddIndex(Index&, bool isSparse); - void NewItem(reindexer::WrSerializer& ser) { scheme_.NewItem(ser, rndGen_); } - const std::vector& GetIndexes() const noexcept { return indexes_; } + void AddIndexToScheme(const Index&, size_t indexNumber); + void NewItem(reindexer::WrSerializer& ser) { scheme_.NewItem(ser, rndGen_, indexes_); } void Dump(std::ostream&) const; private: diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc index 8fb4e9e52..8c027dbc6 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc @@ -1,10 +1,12 @@ #include "ns_scheme.h" #include "core/cjson/jsonbuilder.h" +#include "index.h" +#include "random_generator.h" #include "tools/serializer.h" namespace fuzzing { -void NsScheme::NewItem(reindexer::WrSerializer& ser, RandomGenerator& rnd) { +void NsScheme::NewItem(reindexer::WrSerializer& ser, RandomGenerator& rnd, const std::vector& indexes) { ser.Reset(); if (rnd.RndErr()) { enum Err : uint8_t { Zero, Random, END = Random }; @@ -23,10 +25,195 @@ void NsScheme::NewItem(reindexer::WrSerializer& ser, RandomGenerator& rnd) { } } reindexer::JsonBuilder builder{ser}; - toJson(builder, std::get(ns_.content), rnd); + toJson(builder, std::get(ns_.content), rnd, indexes); } -void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std::string_view name, RandomGenerator& rnd) { +bool NsScheme::IsStruct(const FieldPath& path) const noexcept { + if (path.empty()) return true; + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::holds_alternative(ref[path.back()].content); +} + +bool NsScheme::IsPoint(const FieldPath& path) const noexcept { + if (path.empty()) return false; + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return !std::holds_alternative(ref[path.back()].content) && + std::get(ref[path.back()].content).type == FieldType::Point; +} + +bool NsScheme::isTtl(const std::vector& idxNumbers, const std::vector& indexes) noexcept { + for (size_t idx : idxNumbers) { + assertrx(idx < indexes.size()); + if (indexes[idx].Type() == IndexType::Ttl) { + return true; + } + } + return false; +} + +bool NsScheme::IsTtl(const FieldPath& path, const std::vector& indexes) const noexcept { + if (path.empty()) return false; + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + if (std::holds_alternative(ref[path.back()].content)) { + return false; + } + return isTtl(std::get(ref[path.back()].content).indexes, indexes); +} + +size_t NsScheme::FieldsCount(const FieldPath& path) const noexcept { + if (path.empty()) { + return std::get(ns_.content).size(); + } + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::visit(reindexer::overloaded{[](const Node::Child&) noexcept -> size_t { + assertrx(false); + return 0; + }, + [](const Node::Children& c) noexcept { return c.size(); }}, + ref[path.back()].content); +} + +IsArray NsScheme::IsArray(const FieldPath& path) const noexcept { + if (path.empty()) return ns_.array; + const Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + const auto& idx = (*ptr)[path[i]]; + if (idx.array == IsArray::Yes) return IsArray::Yes; + std::visit( + reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, [](const Node::Child&) noexcept { assertrx(0); }}, + idx.content); + } + assertrx(ptr->size() > path.back()); + return (*ptr)[path.back()].array; +} + +FieldType NsScheme::GetFieldType(const FieldPath& path) const noexcept { + assertrx(!path.empty()); + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::visit(reindexer::overloaded{[](const Node::Child& c) noexcept { return c.type; }, + [](const Node::Children&) noexcept { return FieldType::Struct; }}, + ref[path.back()].content); +} + +void NsScheme::SetFieldType(const FieldPath& path, FieldType ft) noexcept { + assertrx(!path.empty()); + Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::visit(reindexer::overloaded{[ft](Node::Child& c) noexcept { c.type = ft; }, [](Node::Children&) noexcept { assertrx(0); }}, + ref[path.back()].content); +} + +std::string NsScheme::GetJsonPath(const FieldPath& path) const noexcept { + if (path.empty()) return {}; + std::string res; + const Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + const auto& idx = (*ptr)[path[i]]; + res += idx.name; + std::visit( + reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, [](const Node::Child&) noexcept { assertrx(0); }}, + idx.content); + res += '.'; + } + assertrx(ptr->size() > path.back()); + res += (*ptr)[path.back()].name; + return res; +} + +void NsScheme::AddIndex(const FieldPath& path, size_t index, IsSparse isSparse) { + assertrx(!path.empty()); + if (isSparse == IsSparse::No) { + ns_.sparse = IsSparse::No; + } + Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + if (isSparse == IsSparse::No) { + (*ptr)[path[i]].sparse = IsSparse::No; + } + std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assertrx(0); }}, + (*ptr)[path[i]].content); + } + assertrx(ptr->size() > path.back()); + addIndex((*ptr)[path.back()], index, isSparse); +} + +FieldPath NsScheme::AddRndPkField(RandomGenerator& rnd) { + auto& children = std::get(ns_.content); + children.emplace_back(Node{rnd.FieldName(generatedNames_), Node::Child{rnd.RndPkIndexFieldType()}}); + children.back().array = IsArray::No; + children.back().sparse = IsSparse::No; + return {children.size() - 1}; +} + +void NsScheme::addIndex(Node& node, size_t index, IsSparse isSparse) { + if (isSparse == IsSparse::No) { + node.sparse = IsSparse::No; + } + std::visit(reindexer::overloaded{[index](Node::Child& c) noexcept { c.indexes.push_back(index); }, + [](Node::Children&) noexcept { assertrx(0); }}, + node.content); +} + +void NsScheme::fillChildren(Node::Children& children, RandomGenerator& rnd, unsigned level, bool& canBeArray, bool& canBeSparse) { + const size_t fieldsCount = rnd.FieldsCount(level == 0); + children.reserve(fieldsCount); + for (size_t i = 0; i < fieldsCount; ++i) { + auto fName = rnd.FieldName(generatedNames_); + const auto type = rnd.RndFieldType(level); + if (type == FieldType::Struct) { + children.emplace_back(Node{std::move(fName), Node::Children{}}); + fillChildren(std::get(children.back().content), rnd, level + 1, canBeArray, canBeSparse); + if (canBeArray || rnd.RndErr()) { + children.back().array = rnd.RndArrayField(); + } + if (!canBeSparse && !rnd.RndErr()) { + children.back().sparse = IsSparse::No; + } + } else { + children.emplace_back(Node{std::move(fName), Node::Child{type}}); + if (type == FieldType::Point) { + canBeSparse = false; + canBeArray = false; + children.back().sparse = IsSparse::No; + } + if (canBeArray || rnd.RndErr()) { + children.back().array = rnd.RndArrayField(); + } + } + } +} + +const NsScheme::Node::Children& NsScheme::findLastContainer(const FieldPath& path) const noexcept { + const Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + std::visit( + reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, [](const Node::Child&) noexcept { assertrx(0); }}, + (*ptr)[path[i]].content); + } + return *ptr; +} + +NsScheme::Node::Children& NsScheme::findLastContainer(const FieldPath& path) noexcept { + Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assertrx(0); }}, + (*ptr)[path[i]].content); + } + return *ptr; +} + +void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std::string_view name, const std::vector& idxNumbers, + const std::vector& indexes, RandomGenerator& rnd) { switch (ft) { case FieldType::Bool: builder.Put(name, rnd.RndBool(0.5)); @@ -35,7 +222,11 @@ void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std builder.Put(name, rnd.RndIntValue()); break; case FieldType::Int64: - builder.Put(name, rnd.RndInt64Value()); + if (isTtl(idxNumbers, indexes)) { + builder.Put(name, rnd.RndTtlValue()); + } else { + builder.Put(name, rnd.RndInt64Value()); + } break; case FieldType::Double: builder.Put(name, rnd.RndDoubleValue()); @@ -54,41 +245,44 @@ void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std Node::Children children; fillChildren(children, rnd, 2, canBeArray, canBeSparse); auto obj = builder.Object(name); - toJson(obj, children, rnd); + toJson(obj, children, rnd, indexes); } break; default: assertrx(0); } } -void NsScheme::toJson(reindexer::JsonBuilder& builder, const Node::Children& children, RandomGenerator& rnd) { +void NsScheme::toJson(reindexer::JsonBuilder& builder, const Node::Children& children, RandomGenerator& rnd, + const std::vector& indexes) { for (const Node& n : children) { if (!rnd.NeedThisNode(n.sparse)) continue; - if (rnd.RndArrayField(n.array)) { + if (rnd.RndArrayField(n.array) == IsArray::Yes) { auto arr = builder.Array(n.name); const size_t arrSize = rnd.ArraySize(); for (size_t i = 0; i < arrSize; ++i) { if (rnd.RndErr()) { - rndValueToJson(arr, rnd.RndFieldType(), {}, rnd); + rndValueToJson(arr, rnd.RndFieldType(), {}, {}, indexes, rnd); } else { - std::visit(reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(arr, c.type, {}, rnd); }, - [&](const Node::Children& c) { - auto obj = arr.Object(); - toJson(obj, c, rnd); - }}, - n.content); + std::visit( + reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(arr, c.type, {}, c.indexes, indexes, rnd); }, + [&](const Node::Children& c) { + auto obj = arr.Object(); + toJson(obj, c, rnd, indexes); + }}, + n.content); } } } else { if (rnd.RndErr()) { - rndValueToJson(builder, rnd.RndFieldType(), n.name, rnd); + rndValueToJson(builder, rnd.RndFieldType(), n.name, {}, indexes, rnd); } else { - std::visit(reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(builder, c.type, n.name, rnd); }, - [&](const Node::Children& c) { - auto obj = builder.Object(n.name); - toJson(obj, c, rnd); - }}, - n.content); + std::visit( + reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(builder, c.type, n.name, c.indexes, indexes, rnd); }, + [&](const Node::Children& c) { + auto obj = builder.Object(n.name); + toJson(obj, c, rnd, indexes); + }}, + n.content); } } } @@ -100,9 +294,9 @@ void NsScheme::Node::Dump(std::ostream& os, size_t offset) const { for (size_t i = 0; i <= offset; ++i) os << " "; os << "name: " << name << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "sparse: " << (sparse ? "true" : "false") << '\n'; + os << "sparse: " << std::boolalpha << (sparse == IsSparse::Yes) << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "array: " << (array ? "true" : "false") << '\n'; + os << "array: " << std::boolalpha << (array == IsArray::Yes) << '\n'; std::visit(reindexer::overloaded{[&](const Child& child) { for (size_t i = 0; i <= offset; ++i) os << " "; os << "type: " << child.type << '\n'; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h index 675173c36..a63a76e38 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h @@ -1,10 +1,10 @@ #pragma once #include +#include #include #include -#include "estl/overloaded.h" -#include "random_generator.h" +#include "types.h" namespace reindexer { @@ -15,17 +15,22 @@ class JsonBuilder; namespace fuzzing { +class RandomGenerator; +class Index; + class NsScheme { struct Node { using Children = std::vector; struct Child { + Child(FieldType t) noexcept : type{t} {} FieldType type; + std::vector indexes; }; std::string name; std::variant content; - bool sparse{true}; - bool array{false}; + IsSparse sparse{IsSparse::Yes}; + IsArray array{IsArray::No}; void Dump(std::ostream&, size_t offset) const; }; @@ -34,164 +39,29 @@ class NsScheme { bool canBeArray = true, canBeSparse = true; fillChildren(std::get(ns_.content), rnd, 0, canBeArray, canBeSparse); } - size_t FieldsCount(const FieldPath& path) const noexcept { - if (path.empty()) { - return std::get(ns_.content).size(); - } - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::visit(reindexer::overloaded{[](const Node::Child&) noexcept -> size_t { - assertrx(false); - return 0; - }, - [](const Node::Children& c) noexcept { return c.size(); }}, - ref[path.back()].content); - } - bool IsStruct(const FieldPath& path) const noexcept { - if (path.empty()) return true; - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::holds_alternative(ref[path.back()].content); - } - bool IsPoint(const FieldPath& path) const noexcept { - if (path.empty()) return false; - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return !std::holds_alternative(ref[path.back()].content) && - std::get(ref[path.back()].content).type == FieldType::Point; - } - bool IsArray(const FieldPath& path) const noexcept { - if (path.empty()) return ns_.array; - const Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - const auto& idx = (*ptr)[path[i]]; - if (idx.array) return true; - std::visit(reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, - [](const Node::Child&) noexcept { assert(0); }}, - idx.content); - } - assertrx(ptr->size() > path.back()); - return (*ptr)[path.back()].array; - } - FieldType GetFieldType(const FieldPath& path) const noexcept { - assertrx(!path.empty()); - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::visit(reindexer::overloaded{[](const Node::Child& c) noexcept { return c.type; }, - [](const Node::Children&) noexcept { return FieldType::Struct; }}, - ref[path.back()].content); - } - void SetFieldType(const FieldPath& path, FieldType ft) noexcept { - assertrx(!path.empty()); - Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::visit( - reindexer::overloaded{[ft](Node::Child& c) noexcept { c.type = ft; }, [](Node::Children&) noexcept { assert(0); }}, - ref[path.back()].content); - } - std::string GetJsonPath(const FieldPath& path) const noexcept { - if (path.empty()) return {}; - std::string res; - const Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - const auto& idx = (*ptr)[path[i]]; - res += idx.name; - std::visit(reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, - [](const Node::Child&) noexcept { assert(0); }}, - idx.content); - res += '.'; - } - assertrx(ptr->size() > path.back()); - res += (*ptr)[path.back()].name; - return res; - } - void AddIndex(const FieldPath& path, bool isSparse) { - if (path.empty()) return; - if (!isSparse) ns_.sparse = false; - Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - if (!isSparse) { - (*ptr)[path[i]].sparse = false; - } - std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assert(0); }}, - (*ptr)[path[i]].content); - } - assertrx(ptr->size() > path.back()); - mark((*ptr)[path.back()], isSparse); - } - void NewItem(reindexer::WrSerializer&, RandomGenerator&); + size_t FieldsCount(const FieldPath&) const noexcept; + bool IsStruct(const FieldPath&) const noexcept; + bool IsPoint(const FieldPath&) const noexcept; + bool IsTtl(const FieldPath&, const std::vector&) const noexcept; + enum IsArray IsArray(const FieldPath&) const noexcept; + FieldType GetFieldType(const FieldPath&) const noexcept; + void SetFieldType(const FieldPath&, FieldType) noexcept; + std::string GetJsonPath(const FieldPath&) const noexcept; + void AddIndex(const FieldPath&, size_t index, IsSparse); + void NewItem(reindexer::WrSerializer&, RandomGenerator&, const std::vector&); void Dump(std::ostream& os, size_t offset) const { ns_.Dump(os, offset); } - FieldPath AddRndPkField(RandomGenerator& rnd) { - auto& children = std::get(ns_.content); - children.emplace_back(Node{rnd.FieldName(generatedNames_), Node::Child{rnd.RndPkIndexFieldType()}}); - children.back().array = false; - children.back().sparse = false; - return {children.size() - 1}; - } + FieldPath AddRndPkField(RandomGenerator&); private: - static void mark(Node& node, bool isSparse) { - if (!isSparse) { - node.sparse = false; - } - std::visit(reindexer::overloaded{[](Node::Child&) noexcept {}, - [isSparse](Node::Children& c) noexcept { - for (Node& n : c) mark(n, isSparse); - }}, - node.content); - } - void fillChildren(Node::Children& children, RandomGenerator& rnd, unsigned level, bool& canBeArray, bool& canBeSparse) { - const size_t fieldsCount = rnd.FieldsCount(level == 0); - children.reserve(fieldsCount); - for (size_t i = 0; i < fieldsCount; ++i) { - auto fName = rnd.FieldName(generatedNames_); - const auto type = rnd.RndFieldType(level); - if (type == FieldType::Struct) { - children.emplace_back(Node{std::move(fName), Node::Children{}}); - fillChildren(std::get(children.back().content), rnd, level + 1, canBeArray, canBeSparse); - if (canBeArray || rnd.RndErr()) { - children.back().array = rnd.RndArrayField(); - } - if (!canBeSparse && !rnd.RndErr()) { - children.back().sparse = false; - } - } else { - children.emplace_back(Node{std::move(fName), Node::Child{type}}); - if (type == FieldType::Point) { - canBeSparse = false; - canBeArray = false; - children.back().sparse = false; - } - if (canBeArray || rnd.RndErr()) { - children.back().array = rnd.RndArrayField(); - } - } - } - } - const Node::Children& findLastContainer(const FieldPath& path) const noexcept { - const Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - std::visit(reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, - [](const Node::Child&) noexcept { assert(0); }}, - (*ptr)[path[i]].content); - } - return *ptr; - } - Node::Children& findLastContainer(const FieldPath& path) noexcept { - Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assert(0); }}, - (*ptr)[path[i]].content); - } - return *ptr; - } - void toJson(reindexer::JsonBuilder&, const Node::Children&, RandomGenerator&); - void rndValueToJson(reindexer::JsonBuilder&, FieldType, std::string_view name, RandomGenerator&); + static void addIndex(Node&, size_t index, IsSparse); + void fillChildren(Node::Children&, RandomGenerator&, unsigned level, bool& canBeArray, bool& canBeSparse); + const Node::Children& findLastContainer(const FieldPath&) const noexcept; + Node::Children& findLastContainer(const FieldPath&) noexcept; + void toJson(reindexer::JsonBuilder&, const Node::Children&, RandomGenerator&, const std::vector&); + void rndValueToJson(reindexer::JsonBuilder&, FieldType, std::string_view name, const std::vector& idxNumbers, + const std::vector&, RandomGenerator&); + static bool isTtl(const std::vector& idxNumbers, const std::vector&) noexcept; + Node ns_; std::unordered_set generatedNames_; }; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc index 520c3abe7..da0f1d081 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc @@ -1,12 +1,13 @@ #include "query_generator.h" #include "core/query/query.h" +#include "index.h" +#include "ns.h" namespace fuzzing { reindexer::Query QueryGenerator::operator()() { if (namespaces_.empty() || rndGen_.RndErr()) { - std::unordered_set generatedNames; - return reindexer::Query{rndGen_.NsName(generatedNames)}; + return reindexer::Query{rndGen_.GenerateNsName()}; } const auto& ns = rndGen_.RndWhich(namespaces_); reindexer::Query query{ns.GetName()}; @@ -15,21 +16,24 @@ reindexer::Query QueryGenerator::operator()() { case Index: if (const auto& indexes = ns.GetIndexes(); !indexes.empty()) { const auto& idx = rndGen_.RndWhich(indexes); - std::visit(reindexer::overloaded{[&](const Index::Child& c) { rndGen_.RndWhere(query, idx.name, {c.type}); }, + std::visit(reindexer::overloaded{[&](const Index::Child& c) { rndGen_.RndWhere(query, idx.Name(), c.type, idx.Type()); }, [&](const Index::Children& c) { std::vector types; types.reserve(c.size()); for (const auto& child : c) types.push_back(child.type); - rndGen_.RndWhere(query, idx.name, types); + rndGen_.RndWhereComposite(query, idx.Name(), std::move(types), idx.Type()); }}, - idx.content); + idx.Content()); } break; case Field: { const auto path = rndGen_.RndField(ns.GetScheme()); const FieldType type = ns.GetScheme().GetFieldType(path); - if (type != FieldType::Struct) { - rndGen_.RndWhere(query, ns.GetScheme().GetJsonPath(path), {type}); + if (type == FieldType::Struct) { // TODO object find + } else { + const std::optional indexType = + ns.GetScheme().IsTtl(path, ns.GetIndexes()) ? IndexType::Ttl : std::optional{}; + rndGen_.RndWhere(query, ns.GetScheme().GetJsonPath(path), type, indexType); } } break; case Empty: diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h index 2724b23c5..c2644492b 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h @@ -1,6 +1,6 @@ #pragma once -#include "ns.h" +#include "random_generator.h" namespace reindexer { @@ -10,10 +10,11 @@ class Query; namespace fuzzing { +class Ns; + class QueryGenerator { public: - QueryGenerator(const std::vector& nss, std::ostream& os, RandomGenerator::ErrFactorType errorFactor) - : namespaces_{nss}, rndGen_{os, errorFactor} {} + QueryGenerator(const std::vector& nss, RandomGenerator::ErrFactorType errorFactor) : namespaces_{nss}, rndGen_{errorFactor} {} reindexer::Query operator()(); private: diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc index 25e0f640e..e207a6667 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc @@ -1,17 +1,97 @@ #include "random_generator.h" +#include +#include #include +#include +#include "core/payload/fieldsset.h" #include "core/query/query.h" +#include "index.h" #include "ns_scheme.h" namespace fuzzing { -RandomGenerator::RandomGenerator(std::ostream& os, ErrFactorType errorFactor) - : gen_(std::chrono::system_clock::now().time_since_epoch().count()), errFactor_{errorFactor} { +std::string& RandomGenerator::out() noexcept { + static std::string outStr; + return outStr; +} + +std::unique_ptr& RandomGenerator::in() noexcept { + static std::unique_ptr f; + return f; +} + +void RandomGenerator::SetOut(std::string o) { + ASSERT_TRUE(out().empty()); + ASSERT_FALSE(in()); + out() = std::move(o); + { + std::ifstream f{out()}; + ASSERT_FALSE(f.is_open()) << "File '" << out() << "' already exists"; + } +} + +void RandomGenerator::SetIn(const std::string& i) { + ASSERT_FALSE(in()); + ASSERT_TRUE(out().empty()); + in() = std::make_unique(i); + ASSERT_TRUE(in()->is_open()) << "Cannot open file '" << i << '\''; + in()->exceptions(std::ios_base::badbit | std::ios_base::failbit | std::ios_base::eofbit); +} + +RandomGenerator::RandomEngine RandomGenerator::createRandomEngine() { + if (in()) { + RandomEngine ret; + std::string buf; + std::getline(*in(), buf); + std::istringstream ss{buf}; + ss >> ret; + return ret; + } else { + RandomEngine ret(std::chrono::system_clock::now().time_since_epoch().count()); + if (!out().empty()) { + std::ofstream file{out(), std::ios_base::app}; + if (file.is_open()) { + file.exceptions(std::ios_base::badbit | std::ios_base::failbit | std::ios_base::eofbit); + file << ret << std::endl; + } else { + EXPECT_TRUE(false) << "Cannot open file '" << out() << '\''; + } + } + return ret; + } +} + +RandomGenerator::RandomGenerator(ErrFactorType errorFactor) : gen_{createRandomEngine()}, errFactor_{errorFactor} { assertrx(errFactor_.first < errFactor_.second); errParams_ = {static_cast(errFactor_.second - errFactor_.first), static_cast(errFactor_.first)}; - os << gen_ << std::endl; } -RandomGenerator::RandomGenerator(std::istream& is) { is >> gen_; } + +size_t RandomGenerator::FieldsCount(bool firstLevel) { + if (RndErr()) { + enum Err : uint8_t { Zero, TooMany, END = TooMany }; + switch (RndWhich()) { + case Zero: + return 0; + case TooMany: + return RndInt(0, 10'000); + default: + assertrx(0); + } + } + if (firstLevel) { + enum Size : uint8_t { Normal, Long, END = Long }; + switch (RndWhich()) { + case Normal: + return RndInt(1, 9); + case Long: + return RndInt(10, 100); + default: + assertrx(false); + std::abort(); + } + } + return RndInt(1, 5); +} std::string RandomGenerator::FieldName(std::unordered_set& generatedNames) { // TODO static constexpr char alfas[] = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; @@ -148,7 +228,7 @@ FieldPath RandomGenerator::RndScalarField(const NsScheme& nsScheme) { const int end = idx + size; while (idx < end) { res.back() = idx % size; - if (!nsScheme.IsArray(res) && !nsScheme.IsPoint(res)) break; + if (nsScheme.IsArray(res) == IsArray::No && !nsScheme.IsPoint(res)) break; ++idx; } if (idx == end) return {}; @@ -156,7 +236,7 @@ FieldPath RandomGenerator::RndScalarField(const NsScheme& nsScheme) { return res; } -std::string RandomGenerator::IndexFieldType(fuzzing::FieldType ft) { +std::string RandomGenerator::IndexFieldType(FieldType ft) { static const std::string types[] = {"bool", "int", "int64", "double", "string", "uuid", "point", "composite"}; if (RndErr()) { // TODO rnd string @@ -167,52 +247,158 @@ std::string RandomGenerator::IndexFieldType(fuzzing::FieldType ft) { return types[i]; } -std::string RandomGenerator::RndIndexType(fuzzing::FieldType ft, bool pk) { - static const std::string types[] = {"-", "hash", "tree", "ttl", "text", "fuzzytext", "rtree"}; - static const std::vector availableTypes[] = { - {0}, // Bool - {0, 1, 2}, // Int - {0, 1, 2, 3}, // Int64 - {0, 2}, // Double - {0, 1, 2 /*, 4, 5*/}, // String // TODO FT indexes - {1}, // Uuid - {6}, // Point - {1, 2 /*, 4, 5*/} // Struct // TODO FT indexes +IndexType RandomGenerator::RndIndexType(IndexType it) { + if (RndErr()) { + return RndWhich(); // TODO + } + return it; +} + +template * Availables> +IndexType RandomGenerator::rndIndexType(const std::vector& fieldTypes) { + if (RndErr()) { + // TODO rnd string + return RndWhich(); // TODO + } + assertrx(!fieldTypes.empty()); + std::vector availables; + { + const size_t f = static_cast(fieldTypes[0]); + assertrx(f < N); + availables = Availables[f]; + } + for (size_t i = 1, s = fieldTypes.size(); i < s; ++i) { + const size_t f = static_cast(fieldTypes[i]); + std::vector tmp; + tmp.reserve(availables.size()); + assertrx(f < N); + std::set_intersection(availables.begin(), availables.end(), Availables[f].begin(), Availables[f].end(), std::back_inserter(tmp)); + availables = tmp; + } + if (availables.empty()) { + return RndWhich(); // TODO + } else { + return RndWhich(availables); + } +} + +IndexType RandomGenerator::RndIndexType(const std::vector& fieldTypes) { + static const std::vector availableTypes[] = { + {IndexType::Store}, // Bool + {IndexType::Store, IndexType::Hash, IndexType::Tree}, // Int + {IndexType::Store, IndexType::Hash, IndexType::Tree, IndexType::Ttl}, // Int64 + {IndexType::Store, IndexType::Tree}, // Double + {IndexType::Store, IndexType::Hash, IndexType::Tree}, // String // TODO IndexType::FastFT IndexType::FuzzyFT + {IndexType::Hash}, // Uuid + {IndexType::RTree}, // Point + {IndexType::Hash, IndexType::Tree} // Struct // TODO IndexType::FastFT IndexType::FuzzyFT }; - static const std::vector availablePkTypes[] = { - {}, // Bool - {1, 2}, // Int - {1, 2, 3}, // Int64 - {2}, // Double - {1, 2}, // String - {1}, // Uuid - {}, // Point - {1, 2} // Struct + return rndIndexType, availableTypes>(fieldTypes); +} + +IndexType RandomGenerator::RndPkIndexType(const std::vector& fieldTypes) { + static const std::vector availablePkTypes[] = { + {}, // Bool + {IndexType::Hash, IndexType::Tree}, // Int + {IndexType::Hash, IndexType::Tree, IndexType::Ttl}, // Int64 + {IndexType::Tree}, // Double + {IndexType::Hash, IndexType::Tree}, // String + {IndexType::Hash}, // Uuid + {}, // Point + {IndexType::Hash, IndexType::Tree} // Struct }; + return rndIndexType, availablePkTypes>(fieldTypes); +} + +size_t RandomGenerator::ArraySize() { + if (RndErr()) return RndInt(0, 100'000); + enum Size : uint8_t { Short, Normal, Long, VeryLong, END = VeryLong }; + switch (RndWhich()) { + case Short: + return RndInt(0, 5); + case Normal: + return RndInt(6, 20); + case Long: + return RndInt(21, 200); + case VeryLong: + return RndInt(201, 10'000); + default: + assertrx(false); + std::abort(); + } +} + +size_t RandomGenerator::IndexesCount() { if (RndErr()) { - // TODO rnd string - return RndWhich(types); + enum Err : uint8_t { Zero, TooMany, END = TooMany }; + switch (RndWhich()) { + case Zero: + return 0; + case TooMany: + return RndInt(reindexer::kMaxIndexes, 5 + reindexer::kMaxIndexes); + default: + assertrx(0); + } } - const size_t i = static_cast(ft); - size_t n; - if (pk) { - assertrx(i < std::size(availablePkTypes)); - if (availablePkTypes[i].empty()) { - return RndWhich(types); + enum Count : uint8_t { Few, Normal, Many, TooMany, END = TooMany }; + switch (RndWhich()) { + case Few: + return RndInt(1, 3); + case Normal: + return RndInt(4, 20); + case Many: + return RndInt(21, 63); + case TooMany: + return RndInt(64, reindexer::kMaxIndexes); + default: + assertrx(false); + std::abort(); + } +} + +size_t RandomGenerator::compositeIndexSize(size_t scalarIndexesCount) { + if (RndErr()) { + enum Err : uint8_t { Zero, /*One,*/ TooMany, END = TooMany }; + switch (RndWhich()) { + case Zero: + return 0; + /*case One: + return 1;*/ + case TooMany: + return RndInt(0, 10'000); + default: + assertrx(0); + } + } + assertrx(scalarIndexesCount >= 1); + return RndInt(1, scalarIndexesCount); +} + +std::vector RandomGenerator::RndFieldsForCompositeIndex(const std::vector& scalarIndexes) { + std::vector result; + const size_t count = compositeIndexSize(scalarIndexes.size()); + result.reserve(count); + const bool uniqueFields = count <= scalarIndexes.size() && !RndErr(); + // TODO unexisted and not indexed fields + if (uniqueFields) { + auto scalars = scalarIndexes; + while (result.size() < count) { + const size_t idx = rndSize(0, scalars.size() - 1); + result.push_back(scalars[idx]); + scalars.erase(scalars.begin() + idx); } - n = RndWhich(availablePkTypes[i]); } else { - assertrx(i < std::size(availableTypes)); - n = RndWhich(availableTypes[i]); + while (result.size() < count) { + result.push_back(scalarIndexes[rndSize(0, scalarIndexes.size() - 1)]); + } } - assertrx(n < std::size(types)); - return types[n]; + return result; } template <> constexpr size_t RandomGenerator::itemsCount = CondType::CondDWithin + 1; -CondType RandomGenerator::rndCond(fuzzing::FieldType ft) { // TODO array +CondType RandomGenerator::rndCond(FieldType ft) { // TODO array if (RndErr()) { return RndWhich(); } @@ -240,8 +426,6 @@ std::string RandomGenerator::rndStrUuidValue(bool noErrors) { if (!noErrors && RndErr()) { err = RndWhich(); } - std::string res; - if (err == Empty) return res; size_t size = 32; switch (err) { case Short: @@ -253,15 +437,17 @@ std::string RandomGenerator::rndStrUuidValue(bool noErrors) { case TooLong: size = RndInt(51, 100'000); break; - case NoErrors: case Empty: + return {}; + case NoErrors: case WrongVariant: case WrongChar: break; default: - assert(0); + assertrx(0); abort(); } + std::string res; res.reserve(size + 4); if (RndBool(0.001)) { res = std::string(std::string::size_type{size}, '0'); @@ -293,94 +479,127 @@ std::string RandomGenerator::rndStrUuidValue(bool noErrors) { reindexer::Uuid RandomGenerator::rndUuidValue() { return reindexer::Uuid{rndStrUuidValue(true)}; } -void RandomGenerator::RndWhere(reindexer::Query& query, const std::string& field, - const std::vector& types) { // TODO array +int64_t RandomGenerator::RndTtlValue() { + const int64_t now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + // TODO uncomment this after TTL subscribe done + /*enum Size : uint8_t { Negative, FarPast, Past, Now, Future, FarFuture, AnyShort, Any, END = Any }; + switch (RndWhich()) { case Negative: return rndInt64(std::numeric_limits::min(), 0); case FarPast: + return rndInt64(0, now - 10'000); + case Past: + return rndInt64(now - 10'000, now - 10); + case Now: + return rndInt64(now - 10, now + 10); + case Future: + return rndInt64(now + 10, now + 10'000); + case FarFuture: + return rndInt64(now + 10'000, std::numeric_limits::max()); + case AnyShort: + return rndInt64(-50, 50); + case Any: + return rndInt64(std::numeric_limits::min(), std::numeric_limits::max()); + default: + assertrx(false); + std::abort(); + }*/ + return rndInt64(now + 10'000, std::numeric_limits::max()); +} + +void RandomGenerator::RndWhere(reindexer::Query& query, const std::string& field, FieldType fieldType, + std::optional indexType) { // TODO array + if (RndErr()) { + return RndWhereComposite(query, field, RndFieldTypesArray({fieldType}), indexType); + } std::unordered_set generatedNames; - assertrx(!types.empty()); const std::string fldName = FieldName(field, generatedNames); - const auto type = types.size() > 1 ? fuzzing::FieldType::Struct : types[0]; - const auto cond = rndCond(type); - switch (RndFieldType(type)) { - case fuzzing::FieldType::Bool: + const auto cond = rndCond(fieldType); + switch (RndFieldType(fieldType)) { + case FieldType::Bool: query.Where(fldName, cond, RndBool(0.5)); break; - case fuzzing::FieldType::Int: + case FieldType::Int: query.Where(fldName, cond, RndIntValue()); break; - case fuzzing::FieldType::Int64: - query.Where(fldName, cond, RndInt64Value()); + case FieldType::Int64: + if (indexType == IndexType::Ttl) { + query.Where(fldName, cond, RndTtlValue()); + } else { + query.Where(fldName, cond, RndInt64Value()); + } break; - case fuzzing::FieldType::Double: + case FieldType::Double: query.Where(fldName, cond, RndDoubleValue()); break; - case fuzzing::FieldType::String: + case FieldType::String: query.Where(fldName, cond, RndStringValue()); break; - case fuzzing::FieldType::Uuid: + case FieldType::Uuid: if (RndBool(0.5)) { query.Where(fldName, cond, rndUuidValue()); } else { query.Where(fldName, cond, rndStrUuidValue(false)); } break; - case fuzzing::FieldType::Point: + case FieldType::Point: query.Where(fldName, cond, {reindexer::Variant{reindexer::Point{RndDoubleValue(), RndDoubleValue()}}, reindexer::Variant{RndErr() ? RndDoubleValue() : std::abs(RndDoubleValue())}}); break; - case fuzzing::FieldType::Struct: // TODO - if (type == fuzzing::FieldType::Struct) { - } else { - } + case FieldType::Struct: // TODO break; default: assertrx(0); } } -std::ostream& operator<<(std::ostream& os, FieldType ft) { - switch (ft) { - case FieldType::Bool: - return os << "bool"; - case FieldType::Int: - return os << "int"; - case FieldType::Int64: - return os << "int64"; - case FieldType::Double: - return os << "double"; - case FieldType::String: - return os << "string"; - case FieldType::Uuid: - return os << "uuid"; - case FieldType::Point: - return os << "point"; - case FieldType::Struct: - return os << "struct"; - default: - assertrx(0); +void RandomGenerator::RndWhereComposite(reindexer::Query& query, const std::string& field, std::vector&& fieldTypes, + std::optional indexType) { // TODO array + if (RndErr()) { + return RndWhere(query, field, RndFieldType(), indexType); } - return os; -} - -reindexer::KeyValueType ToKeyValueType(FieldType ft) { - switch (ft) { - case FieldType::Bool: - return reindexer::KeyValueType::Bool{}; - case FieldType::Int: - return reindexer::KeyValueType::Int{}; - case FieldType::Int64: - return reindexer::KeyValueType::Int64{}; - case FieldType::Double: - return reindexer::KeyValueType::Double{}; - case FieldType::String: - return reindexer::KeyValueType::String{}; - case FieldType::Uuid: - return reindexer::KeyValueType::Uuid{}; - case FieldType::Point: - case FieldType::Struct: - default: - assertrx(0); + std::unordered_set generatedNames; + const std::string fldName = FieldName(field, generatedNames); + fieldTypes = RndFieldTypesArray(std::move(fieldTypes)); + const auto cond = rndCond(FieldType::Struct); + reindexer::VariantArray keys; + keys.reserve(fieldTypes.size()); + for (const FieldType ft : fieldTypes) { + switch (ft) { + case FieldType::Bool: + keys.emplace_back(RndBool(0.5)); + break; + case FieldType::Int: + keys.emplace_back(RndIntValue()); + break; + case FieldType::Int64: + if (indexType == IndexType::Ttl) { + keys.emplace_back(RndTtlValue()); + } else { + keys.emplace_back(RndInt64Value()); + } + break; + case FieldType::Double: + keys.emplace_back(RndDoubleValue()); + break; + case FieldType::String: + keys.emplace_back(RndStringValue()); + break; + case FieldType::Uuid: + if (RndBool(0.5)) { + keys.emplace_back(rndUuidValue()); + } else { + keys.emplace_back(rndStrUuidValue(false)); + } + break; + case FieldType::Point: + keys.emplace_back(reindexer::Point{RndDoubleValue(), RndDoubleValue()}); + break; + case FieldType::Struct: // TODO + break; + default: + assertrx(0); + } } + query.WhereComposite(fldName, cond, {std::move(keys)}); } } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h index 7a628fe85..4960f28c3 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h @@ -1,10 +1,12 @@ #pragma once -#include +#include +#include #include #include #include "core/type_consts.h" #include "tools/assertrx.h" +#include "types.h" namespace reindexer { @@ -16,137 +18,69 @@ class KeyValueType; namespace fuzzing { -struct Index; class NsScheme; -enum class FieldType { Bool, Int, Int64, Double, String, Uuid, Point, Struct, END = Struct }; -reindexer::KeyValueType ToKeyValueType(FieldType); -std::ostream& operator<<(std::ostream&, FieldType); -using FieldPath = std::vector; - class RandomGenerator { using ErrFactorInt = uint32_t; public: using ErrFactorType = std::pair; - RandomGenerator(std::ostream&, ErrFactorType errorFactor); - RandomGenerator(std::istream&); + RandomGenerator(ErrFactorType errorFactor); - size_t FieldsCount(bool firstLevel) { - if (RndErr()) { - enum Err : uint8_t { Zero, TooMany, END = TooMany }; - switch (RndWhich()) { - case Zero: - return 0; - case TooMany: - return RndInt(0, 10'000); - default: - assertrx(0); - } - } - if (firstLevel) { - enum Size : uint8_t { Normal, Long, END = Long }; - switch (RndWhich()) { - case Normal: - return RndInt(1, 9); - case Long: - return RndInt(10, 100); - default: - assertrx(false); - std::abort(); - } - } - return RndInt(1, 5); - } - fuzzing::FieldType RndFieldType(unsigned level) { + size_t FieldsCount(bool firstLevel); + FieldType RndFieldType(unsigned level) { const bool withoutStruct = level > 2 && (level > 5 || !RndBool(1.0 / (2 << (2 * level)))); - return static_cast( - RndInt(static_cast(fuzzing::FieldType::Bool), static_cast(fuzzing::FieldType::Struct) - withoutStruct)); + return static_cast(RndInt(static_cast(FieldType::Bool), static_cast(FieldType::Struct) - withoutStruct)); } - fuzzing::FieldType RndFieldType() { - return static_cast( - RndInt(static_cast(fuzzing::FieldType::Bool), static_cast(fuzzing::FieldType::Point))); + FieldType RndFieldType() { + return static_cast(RndInt(static_cast(FieldType::Bool), static_cast(FieldType::Point))); } - fuzzing::FieldType RndPkIndexFieldType() { - return static_cast( - RndInt(static_cast(fuzzing::FieldType::Int), static_cast(fuzzing::FieldType::Uuid))); + FieldType RndPkIndexFieldType() { + return static_cast(RndInt(static_cast(FieldType::Int), static_cast(FieldType::Uuid))); } - std::string IndexFieldType(fuzzing::FieldType); - fuzzing::FieldType RndFieldType(fuzzing::FieldType type) { + std::string IndexFieldType(FieldType); + FieldType RndFieldType(FieldType type) { if (RndErr()) { - return RndWhich(); + return RndWhich(); } return type; } - std::string RndIndexType(fuzzing::FieldType, bool pk); - bool RndArrayField() { return RndBool(0.2); } - bool RndArrayField(bool array) { return RndErr() ? !array : array; } - size_t ArraySize() { - if (RndErr()) return RndInt(0, 100'000); - enum Size : uint8_t { Short, Normal, Long, VeryLong, END = VeryLong }; - switch (RndWhich()) { - case Short: - return RndInt(0, 5); - case Normal: - return RndInt(6, 20); - case Long: - return RndInt(21, 200); - case VeryLong: - return RndInt(201, 10'000); - default: - assertrx(false); - std::abort(); + std::vector RndFieldTypesArray(std::vector&& types) { + if (!RndErr()) { + return std::move(types); } - } - bool PkIndex(bool pk) { return RndErr() ? RndBool(0.5) : pk; } - bool SparseIndex(bool pk) { return pk ? RndErr() : RndBool(0.2); } - bool DenseIndex() { return RndBool(0.2); } - int64_t ExpiredIndex() { return RndInt(0, 100'000); } // TODO - size_t IndexesCount() { - if (RndErr()) { - enum Err : uint8_t { Zero, TooMany, END = TooMany }; - switch (RndWhich()) { - case Zero: - return 0; - case TooMany: - return RndInt(0, 1'000); - default: - assertrx(0); - } + if (RndBool(0.5)) { + types.resize(compositeIndexSize(types.size())); } - enum Count : uint8_t { Few, Normal, Many, TooMany, END = TooMany }; - switch (RndWhich()) { - case Few: - return RndInt(1, 3); - case Normal: - return RndInt(4, 6); - case Many: - return RndInt(7, 20); - case TooMany: - return RndInt(21, 63); - default: - assertrx(false); - std::abort(); + for (auto& t : types) { + t = RndFieldType(); } + return std::move(types); } - bool CompositeIndex() { return RndBool(0.2); } - bool UniqueName() { return RndBool(0.5); } - size_t CompositeIndexSize() { + IndexType RndIndexType(const std::vector&); + IndexType RndPkIndexType(const std::vector&); + IndexType RndIndexType(IndexType); + IsArray RndArrayField() { return RndBool(0.2) ? IsArray::Yes : IsArray::No; } + IsArray RndArrayField(IsArray array) { if (RndErr()) { - enum Err : uint8_t { Zero, One, TooMany, END = TooMany }; - switch (RndWhich()) { - case Zero: - return 0; - case One: - return 1; - case TooMany: - return RndInt(0, 10'000); - default: - assertrx(0); - } + return array == IsArray::Yes ? IsArray::No : IsArray::Yes; } - return RndInt(2, 5); + return array; + } + size_t ArraySize(); + bool PkIndex(bool pk) { return RndErr() ? RndBool(0.5) : pk; } + IsSparse RndSparseIndex(FieldType fldType) { + const bool couldBeSparse = fldType != FieldType::Struct && fldType != FieldType::Uuid; // TODO remove uuid #1470 + return (couldBeSparse ? RndBool(0.2) : RndErr()) ? IsSparse::Yes : IsSparse::No; } + bool RndSparseIndex(IsSparse isSparse) { return (isSparse == IsSparse::Yes) != RndErr(); } + bool DenseIndex() { return RndBool(0.2); } + int64_t ExpiredIndex() { return RndInt(0, 100'000); } // TODO + size_t IndexesCount(); + bool CompositeIndex(size_t scalarIndexesCount) { return scalarIndexesCount < 1 ? RndErr() : RndBool(0.2); } + bool UniqueName() { return RndBool(0.5); } + size_t compositeIndexSize(size_t scalarIndexesCount); + std::vector RndFieldsForCompositeIndex(const std::vector& scalarIndexes); std::string FieldName(std::unordered_set& generatedNames); std::string FieldName(const std::string& fieldName, std::unordered_set& generatedNames) { if (RndErr()) return FieldName(generatedNames); @@ -155,9 +89,12 @@ class RandomGenerator { FieldPath RndField(const NsScheme&); FieldPath RndScalarField(const NsScheme&); std::string IndexName(std::unordered_set& generatedNames) { return FieldName(generatedNames); } // TODO - std::string NsName(std::unordered_set& generatedNames) { return FieldName(generatedNames); } // TODO - std::string NsName(const std::string& nsName, std::unordered_set& generatedNames) { - if (RndErr()) return NsName(generatedNames); + std::string GenerateNsName() { // TODO + std::unordered_set generatedNames; + return FieldName(generatedNames); + } + std::string NsName(const std::string& nsName) { + if (RndErr()) return GenerateNsName(); return nsName; } int RndInt(int min, int max) { return rndInt_(gen_, IntRndParams(min, max)); } @@ -200,7 +137,7 @@ class RandomGenerator { return err; } char RndChar() { return rndChar_(gen_); } - bool NeedThisNode(bool sparse) { return sparse ? RndBool(0.5) : !RndErr(); } + bool NeedThisNode(IsSparse sparse) { return sparse == IsSparse::Yes ? RndBool(0.5) : !RndErr(); } int RndIntValue() { enum Size : uint8_t { Short, Long, END = Long }; switch (RndWhich()) { @@ -225,6 +162,7 @@ class RandomGenerator { std::abort(); } } + int64_t RndTtlValue(); bool RndBool(double p) { return rndBool_(gen_, BoolRndParams{p}); } double RndDoubleValue() { enum Size : uint8_t { Short, Long, END = Long }; @@ -273,14 +211,19 @@ class RandomGenerator { } template const auto& RndWhich(const Cont& cont) { - assert(!std::empty(cont)); + assertrx(!std::empty(cont)); auto it = std::begin(cont); std::advance(it, rndSize(0, std::size(cont) - 1)); return *it; } - void RndWhere(reindexer::Query&, const std::string& field, const std::vector&); + void RndWhere(reindexer::Query&, const std::string& field, FieldType, std::optional); + void RndWhereComposite(reindexer::Query&, const std::string& field, std::vector&&, std::optional); + + static void SetOut(std::string); + static void SetIn(const std::string&); private: + using RandomEngine = std::default_random_engine; using IntRndParams = std::uniform_int_distribution<>::param_type; using SizeRndParams = std::uniform_int_distribution::param_type; using Int64RndParams = std::uniform_int_distribution::param_type; @@ -291,11 +234,16 @@ class RandomGenerator { int rndInt(IntRndParams params) { return rndInt_(gen_, params); } int64_t rndInt64(int64_t min, int64_t max) { return rndInt64_(gen_, Int64RndParams(min, max)); } size_t rndSize(size_t min, size_t max) { return rndSize_(gen_, SizeRndParams(min, max)); } - CondType rndCond(fuzzing::FieldType); + CondType rndCond(FieldType); std::string rndStrUuidValue(bool noErrors); reindexer::Uuid rndUuidValue(); + template * Availables> + IndexType rndIndexType(const std::vector&); + static std::string& out() noexcept; + static std::unique_ptr& in() noexcept; + static RandomEngine createRandomEngine(); - std::default_random_engine gen_; + RandomEngine gen_; ErrFactorType errFactor_; ErrorParams errParams_; std::uniform_int_distribution<> rndInt_; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/types.cc b/cpp_src/gtests/tests/fixtures/fuzzing/types.cc new file mode 100644 index 000000000..11b50b658 --- /dev/null +++ b/cpp_src/gtests/tests/fixtures/fuzzing/types.cc @@ -0,0 +1,90 @@ +#include "types.h" + +#include +#include +#include "core/key_value_type.h" + +namespace fuzzing { + +std::ostream& operator<<(std::ostream& os, FieldType ft) { + switch (ft) { + case FieldType::Bool: + return os << "bool"; + case FieldType::Int: + return os << "int"; + case FieldType::Int64: + return os << "int64"; + case FieldType::Double: + return os << "double"; + case FieldType::String: + return os << "string"; + case FieldType::Uuid: + return os << "uuid"; + case FieldType::Point: + return os << "point"; + case FieldType::Struct: + return os << "struct"; + default: + assertrx(0); + } + return os; +} + +reindexer::KeyValueType ToKeyValueType(FieldType ft) { + switch (ft) { + case FieldType::Bool: + return reindexer::KeyValueType::Bool{}; + case FieldType::Int: + return reindexer::KeyValueType::Int{}; + case FieldType::Int64: + return reindexer::KeyValueType::Int64{}; + case FieldType::Double: + return reindexer::KeyValueType::Double{}; + case FieldType::String: + return reindexer::KeyValueType::String{}; + case FieldType::Uuid: + return reindexer::KeyValueType::Uuid{}; + case FieldType::Point: + return reindexer::KeyValueType::Undefined{}; // TODO change to KeyValueType::Point #1352 + case FieldType::Struct: + default: + assertrx(0); + } +} + +std::ostream& operator<<(std::ostream& os, const FieldPath& fp) { + os << '['; + for (size_t i = 0, s = fp.size(); i < s; ++i) { + if (i != 0) { + os << ' '; + } + os << fp[i]; + } + return os << ']' << std::endl; +} + +std::string_view ToText(IndexType it) { + using namespace std::string_view_literals; + switch (it) { + case IndexType::Store: + return "-"sv; + case IndexType::Hash: + return "hash"sv; + case IndexType::Tree: + return "tree"sv; + case IndexType::Ttl: + return "ttl"sv; + case IndexType::FastFT: + return "text"sv; + case IndexType::FuzzyFT: + return "fuzzytext"sv; + case IndexType::RTree: + return "rtree"sv; + default: + assertrx(0); + } +} + +std::ostream& operator<<(std::ostream& os, IndexType it) { return os << ToText(it); } + +} // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/types.h b/cpp_src/gtests/tests/fixtures/fuzzing/types.h new file mode 100644 index 000000000..db97087c6 --- /dev/null +++ b/cpp_src/gtests/tests/fixtures/fuzzing/types.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +namespace reindexer { + +class KeyValueType; + +} // namespace reindexer + +namespace fuzzing { + +enum class FieldType { Bool, Int, Int64, Double, String, Uuid, Point, Struct, END = Struct }; +reindexer::KeyValueType ToKeyValueType(FieldType); +std::ostream& operator<<(std::ostream&, FieldType); + +using FieldPath = std::vector; +std::ostream& operator<<(std::ostream&, const FieldPath&); + +enum class IndexType { Store, Hash, Tree, Ttl, FastFT, FuzzyFT, RTree, END = RTree }; +std::string_view ToText(IndexType); +std::ostream& operator<<(std::ostream&, IndexType); + +enum class IsArray : bool { Yes = true, No = false }; +enum class IsSparse : bool { Yes = true, No = false }; + +} // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/get_pk_api.h b/cpp_src/gtests/tests/fixtures/get_pk_api.h index ebe17f9ea..29e7ad876 100644 --- a/cpp_src/gtests/tests/fixtures/get_pk_api.h +++ b/cpp_src/gtests/tests/fixtures/get_pk_api.h @@ -90,7 +90,7 @@ class ExtractPK : public testing::Test { Error err = db_->Select(query, qres); if (!err.ok()) return ResultType(err, QueryResults{}); - if (print) printQueryResults(query._namespace, qres); + if (print) printQueryResults(query.NsName(), qres); return ResultType(err, std::move(qres)); } diff --git a/cpp_src/gtests/tests/fixtures/grpcclient_api.h b/cpp_src/gtests/tests/fixtures/grpcclient_api.h index 39230d1a8..98b05c866 100644 --- a/cpp_src/gtests/tests/fixtures/grpcclient_api.h +++ b/cpp_src/gtests/tests/fixtures/grpcclient_api.h @@ -266,8 +266,9 @@ class GrpcClientApi : public ReindexerApi { reindexer::WrSerializer wrser; reindexer::Serializer rdser(cjson); - reindexer::CJsonDecoder decoder(const_cast(nsTypes.first)); - ASSERT_NO_THROW(decoder.Decode(pl, rdser, wrser)); + std::deque storage; + reindexer::CJsonDecoder decoder(const_cast(nsTypes.first), storage); + ASSERT_NO_THROW(decoder.Decode<>(pl, rdser, wrser)); ASSERT_TRUE(rdser.Eof()); } diff --git a/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h b/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h index 1c0434473..ba10e1784 100644 --- a/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h +++ b/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h @@ -6,9 +6,6 @@ #include "gason/gason.h" #include "reindexer_api.h" -using reindexer::Item; -using reindexer::ItemImpl; - class ItemMoveSemanticsApi : public ReindexerApi { protected: const std::string pkField = "bookid"; @@ -18,14 +15,22 @@ class ItemMoveSemanticsApi : public ReindexerApi { void SetUp() override { ReindexerApi::SetUp(); - rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); - rt.reindexer->AddIndex(default_namespace, {"bookid", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"title", "text", "string", IndexOpts()}); - rt.reindexer->AddIndex(default_namespace, {"pages", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"price", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"genreid_fk", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"authorid_fk", "hash", "int", IndexOpts().PK()}); - rt.reindexer->Commit(default_namespace); + auto err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"bookid", "hash", "int", IndexOpts().PK()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"title", "text", "string", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"pages", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"price", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"genreid_fk", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"authorid_fk", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->Commit(default_namespace); + ASSERT_TRUE(err.ok()) << err.what(); } void prepareItems() { @@ -48,7 +53,8 @@ class ItemMoveSemanticsApi : public ReindexerApi { ASSERT_TRUE(err.ok()) << err.what(); ASSERT_NO_THROW(gason::JsonParser().Parse(item.GetJSON())); } - rt.reindexer->Commit(default_namespace); + const auto err = rt.reindexer->Commit(default_namespace); + ASSERT_TRUE(err.ok()) << err.what(); } Item getItemById(int id) { diff --git a/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h b/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h index 6326c8faf..8834ba94e 100644 --- a/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h +++ b/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h @@ -23,7 +23,8 @@ class JoinOnConditionsApi : public JoinSelectsApi { builder.End(); err = item.FromJSON(ser.c_str()); ASSERT_TRUE(err.ok()) << err.what(); - rt.reindexer->Insert(leftNs, item); + err = rt.reindexer->Insert(leftNs, item); + ASSERT_TRUE(err.ok()) << err.what(); } for (unsigned int i = 0; i < rightNsData.size(); i++) { @@ -40,7 +41,8 @@ class JoinOnConditionsApi : public JoinSelectsApi { builder.End(); err = item.FromJSON(ser.c_str()); ASSERT_TRUE(err.ok()) << err.what(); - rt.reindexer->Insert(rightNs, item); + err = rt.reindexer->Insert(rightNs, item); + ASSERT_TRUE(err.ok()) << err.what(); } } diff --git a/cpp_src/gtests/tests/fixtures/queries_api.cc b/cpp_src/gtests/tests/fixtures/queries_api.cc index 9047a9627..a1be92c4c 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.cc +++ b/cpp_src/gtests/tests/fixtures/queries_api.cc @@ -281,6 +281,7 @@ void QueriesApi::initConditionsNs() { ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->AddIndex(conditionsNs, {kFieldNameId, "hash", "int", IndexOpts{}.PK()}); ASSERT_TRUE(err.ok()) << err.what(); + addIndexFields(conditionsNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); for (const auto& fit : fieldIndexTypes) { for (const auto& it : fit.indexTypes) { for (const bool isArray : {true, false}) { @@ -291,8 +292,8 @@ void QueriesApi::initConditionsNs() { const std::string fieldType{fit.fieldType.Name()}; const std::string indexName{createIndexName(fieldType, it, isArray, isSparse)}; err = rt.reindexer->AddIndex(conditionsNs, {indexName, it, fieldType, IndexOpts{}.Array(isArray).Sparse(isSparse)}); - addIndexFields(conditionsNs, indexName, {{indexName, fit.fieldType}}); ASSERT_TRUE(err.ok()) << err.what(); + addIndexFields(conditionsNs, indexName, {{indexName, fit.fieldType}}); } } } @@ -300,6 +301,27 @@ void QueriesApi::initConditionsNs() { setPkFields(conditionsNs, {kFieldNameId}); } +void QueriesApi::initUUIDNs() { + const auto err = rt.reindexer->OpenNamespace(uuidNs); + ASSERT_TRUE(err.ok()) << err.what(); + DefineNamespaceDataset( + uuidNs, + { + IndexDeclaration{kFieldNameId, "hash", "int", IndexOpts{}.PK(), 0}, + IndexDeclaration{kFieldNameUuid, "hash", "uuid", IndexOpts{}, 0}, + /*IndexDeclaration{kFieldNameUuidSparse, "hash", "uuid", IndexOpts{}.Sparse(), 0}, // TODO uncomment this #1470 + IndexDeclaration{kFieldNameUuidNotIndex2, "hash", "uuid", IndexOpts{}, 0}, + IndexDeclaration{kFieldNameUuidNotIndex3, "hash", "uuid", IndexOpts{}.Sparse(), 0},*/ + IndexDeclaration{kFieldNameUuidArr, "hash", "uuid", IndexOpts{}.Array(), 0}, + // IndexDeclaration{kFieldNameUuidArrSparse, "hash", "uuid", IndexOpts{}.Array().Sparse(), 0} // TODO uncomment this #1470 + }); + for (const auto& idx : + {kFieldNameUuid, kFieldNameUuidArr /*, kFieldNameUuidSparse, kFieldNameUuidArrSparse*/}) { // TODO uncomment this #1470 + addIndexFields(uuidNs, idx, {{idx, reindexer::KeyValueType::Uuid{}}}); + } + setPkFields(uuidNs, {kFieldNameId}); +} + static reindexer::Variant createRandValue(int id, reindexer::KeyValueType fieldType) { using namespace reindexer; return fieldType.EvaluateOneOf(overloaded{ @@ -376,41 +398,19 @@ static reindexer::VariantArray createRandArrValues(size_t min, size_t max, int i } void QueriesApi::checkAllConditions(const std::string& fieldName, reindexer::KeyValueType fieldType, NullAllowed nullAllowed) { - for (const auto cond : {CondEq, CondSet, CondAllSet, CondLt, CondLe, CondGt, CondGe, CondRange, CondAny, CondEmpty, CondLike}) { - size_t min = 0, max = rand() % kMaxArraySize; - switch (cond) { - case CondEq: - case CondSet: - case CondAllSet: - break; - case CondLike: - if (!fieldType.Is()) { - continue; - } - [[fallthrough]]; - case CondLt: - case CondLe: - case CondGt: - case CondGe: - min = max = 1; - break; - case CondRange: - min = max = 2; - break; - case CondAny: - case CondEmpty: - if (nullAllowed == NullAllowed::No) { - continue; - } - min = max = 0; - break; - case CondDWithin: // TODO #1352 - assert(0); + for (const auto cond : {CondEq, CondSet, CondAllSet, CondLt, CondLe, CondGt, CondGe, CondRange, CondAny, CondEmpty, + CondLike}) { // TODO CondDWithin #1352 + if (cond == CondLike && !fieldType.Is()) { + continue; } + if (nullAllowed == NullAllowed::No && (cond == CondAny || cond == CondEmpty)) { + continue; + } + const auto argsCount = minMaxArgs(cond, 20); for (size_t i = 0; i < 3; ++i) { - ExecuteAndVerify( - reindexer::Query{conditionsNs}.Where(fieldName, cond, createRandArrValues(min, max, rand() % conditionsNsSize, fieldType))); - if (min <= 1 && max >= 1) { + ExecuteAndVerify(reindexer::Query{conditionsNs}.Where( + fieldName, cond, createRandArrValues(argsCount.min, argsCount.max, rand() % conditionsNsSize, fieldType))); + if (argsCount.min <= 1 && argsCount.max >= 1) { ExecuteAndVerify( reindexer::Query{conditionsNs}.Where(fieldName, cond, createRandValue(rand() % conditionsNsSize, fieldType))); } @@ -439,3 +439,99 @@ void QueriesApi::CheckConditions() { checkAllConditions(fieldType + "_array", fit.fieldType, NullAllowed::Yes); } } + +void QueriesApi::FillUUIDNs() { + static size_t lastId = 0; + reindexer::WrSerializer ser; + for (size_t i = lastId; i < uuidNsSize + lastId; ++i) { + Item item = rt.reindexer->NewItem(uuidNs); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + if (rand() % 2) { + ser.Reset(); + { + reindexer::JsonBuilder json{ser}; + json.Put(kFieldNameId, i); + json.Put(kFieldNameUuid, randStrUuid()); + /*if (rand() % 2) { + json.Put(kFieldNameUuidSparse, randStrUuid()); // TODO uncomment this #1470 + }*/ + { + auto arr = json.Array(kFieldNameUuidArr); + for (size_t j = 0, s = rand() % 10; j < s; ++j) { + arr.Put({}, randStrUuid()); + } + } + /*if (rand() % 2) { + auto arr = json.Array(kFieldNameUuidArrSparse); // TODO uncomment this #1470 + for (size_t j = 0, s = rand() % 10; j < s; ++j) { + arr.Put({}, randStrUuid()); + } + }*/ + if (rand() % 2) { + json.Put(kFieldNameUuidNotIndex, randStrUuid()); + } + /*json.Put(kFieldNameUuidNotIndex2, randStrUuid()); // TODO uncomment this #1470 + if (rand() % 2) { + json.Put(kFieldNameUuidNotIndex3, randStrUuid()); + }*/ + if (rand() % 2) { + json.Put(kFieldNameRndString, RandString()); + } + } + const auto err = item.FromJSON(ser.Slice()); + ASSERT_TRUE(err.ok()) << err.what(); + } else { + item[kFieldNameId] = int(i); + if (rand() % 2) { + item[kFieldNameUuid] = randUuid(); + } else { + item[kFieldNameUuid] = randStrUuid(); + } + /*if (rand() % 2) { + item[kFieldNameUuidSparse] = randUuid(); // TODO uncomment this #1470 + }*/ + item[kFieldNameUuidArr] = randHeterogeneousUuidArray(0, 20); + /*if (rand() % 2) { + item[kFieldNameUuidArrSparse] = randHeterogeneousUuidArray(0, 20); // TODO uncomment this #1470 + } + if (rand() % 2) { + item[kFieldNameUuidNotIndex2] = randUuid(); + } else { + item[kFieldNameUuidNotIndex2] = randStrUuid(); + } + if (rand() % 2) { + if (rand() % 2) { + item[kFieldNameUuidNotIndex3] = randUuid(); + } else { + item[kFieldNameUuidNotIndex3] = randStrUuid(); + } + }*/ + } + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + Upsert(uuidNs, item); + saveItem(std::move(item), uuidNs); + } + const auto err = Commit(uuidNs); + ASSERT_TRUE(err.ok()) << err.what(); + lastId += uuidNsSize; +} + +void QueriesApi::CheckUUIDQueries() { + for (size_t i = 0; i < 10; ++i) { + for (const auto& field : { + kFieldNameUuid, kFieldNameUuidArr, kFieldNameUuidNotIndex, kFieldNameRndString /*, + kFieldNameUuidSparse, kFieldNameUuidArrSparse, kFieldNameUuidNotIndex2, kFieldNameUuidNotIndex3*/ + }) { // TODO uncomment this #1470 + for (auto cond : {CondEq, CondLe, CondLt, CondSet, CondGe, CondGt, CondAllSet, CondRange}) { + const auto argsCount = minMaxArgs(cond, 20); + if (argsCount.min <= 1 && argsCount.max >= 1) { + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randUuid())); + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randStrUuid())); + } + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randUuidArray(argsCount.min, argsCount.max))); + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randStrUuidArray(argsCount.min, argsCount.max))); + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randHeterogeneousUuidArray(argsCount.min, argsCount.max))); + } + } + } +} diff --git a/cpp_src/gtests/tests/fixtures/queries_api.h b/cpp_src/gtests/tests/fixtures/queries_api.h index 20e4aa9af..ed95e3a38 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.h +++ b/cpp_src/gtests/tests/fixtures/queries_api.h @@ -9,14 +9,11 @@ #include #include "core/cjson/jsonbuilder.h" #include "core/keyvalue/geometry.h" -#include "core/nsselecter/sortexpression.h" #include "core/queryresults/joinresults.h" -#include "core/type_consts_helpers.h" #include "gtests/tools.h" #include "queries_verifier.h" #include "reindexer_api.h" -#include "tools/random.h" -#include "tools/stringstools.h" +#include "tools/randompoint.h" class QueriesApi : public ReindexerApi, public QueriesVerifier { public: @@ -66,12 +63,33 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kCompositeFieldUuidName, "hash", "composite", IndexOpts{}, 0}, IndexDeclaration{kFieldNameYearSparse, "hash", "string", IndexOpts{}.Sparse(), 0}, }); + addIndexFields(default_namespace, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameGenre, {{kFieldNameGenre, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameYear, {{kFieldNameYear, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNamePackages, {{kFieldNamePackages, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameCountries, {{kFieldNameCountries, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameAge, {{kFieldNameAge, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameDescription, {{kFieldNameDescription, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameRate, {{kFieldNameRate, reindexer::KeyValueType::Double{}}}); + addIndexFields(default_namespace, kFieldNameIsDeleted, {{kFieldNameIsDeleted, reindexer::KeyValueType::Bool{}}}); + addIndexFields(default_namespace, kFieldNameActor, {{kFieldNameActor, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNamePriceId, {{kFieldNamePriceId, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameLocation, {{kFieldNameLocation, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameEndTime, {{kFieldNameEndTime, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameStartTime, {{kFieldNameStartTime, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameBtreeIdsets, {{kFieldNameBtreeIdsets, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameTemp, {{kFieldNameTemp, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameNumeric, {{kFieldNameNumeric, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameUuid, {{kFieldNameUuid, reindexer::KeyValueType::Uuid{}}}); + addIndexFields(default_namespace, kFieldNameUuidArr, {{kFieldNameUuidArr, reindexer::KeyValueType::Uuid{}}}); addIndexFields(default_namespace, kCompositeFieldIdTemp, {{kFieldNameId, reindexer::KeyValueType::Int{}}, {kFieldNameTemp, reindexer::KeyValueType::String{}}}); addIndexFields(default_namespace, kCompositeFieldAgeGenre, {{kFieldNameAge, reindexer::KeyValueType::Int{}}, {kFieldNameGenre, reindexer::KeyValueType::Int{}}}); addIndexFields(default_namespace, kCompositeFieldUuidName, {{kFieldNameUuid, reindexer::KeyValueType::Uuid{}}, {kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameYearSparse, {{kFieldNameYearSparse, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(joinNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -81,6 +99,12 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNameName, "tree", "string", IndexOpts(), 0}, IndexDeclaration{kFieldNameDescription, "text", "string", IndexOpts{}, 0}, IndexDeclaration{kFieldNameYearSparse, "hash", "string", IndexOpts().Sparse(), 0}}); + addIndexFields(joinNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(joinNs, kFieldNameYear, {{kFieldNameYear, reindexer::KeyValueType::Int{}}}); + addIndexFields(joinNs, kFieldNameAge, {{kFieldNameAge, reindexer::KeyValueType::Int{}}}); + addIndexFields(joinNs, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(joinNs, kFieldNameDescription, {{kFieldNameDescription, reindexer::KeyValueType::String{}}}); + addIndexFields(joinNs, kFieldNameYearSparse, {{kFieldNameYearSparse, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(testSimpleNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -90,6 +114,10 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNameName, "hash", "string", IndexOpts(), 0}, IndexDeclaration{kFieldNamePhone, "hash", "string", IndexOpts(), 0}, }); + addIndexFields(testSimpleNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(testSimpleNs, kFieldNameYear, {{kFieldNameYear, reindexer::KeyValueType::Int{}}}); + addIndexFields(testSimpleNs, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(testSimpleNs, kFieldNamePhone, {{kFieldNamePhone, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(compositeIndexesNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -105,6 +133,12 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kCompositeFieldPriceTitle, "hash", "composite", IndexOpts(), 0}, IndexDeclaration{kCompositeFieldPagesTitle, "hash", "composite", IndexOpts(), 0}, IndexDeclaration{kCompositeFieldBookidBookid2, "hash", "composite", IndexOpts().PK(), 0}}); + addIndexFields(compositeIndexesNs, kFieldNameBookid, {{kFieldNameBookid, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNameBookid2, {{kFieldNameBookid2, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNameTitle, {{kFieldNameTitle, reindexer::KeyValueType::String{}}}); + addIndexFields(compositeIndexesNs, kFieldNamePages, {{kFieldNamePages, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNamePrice, {{kFieldNamePrice, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); addIndexFields(compositeIndexesNs, kCompositeFieldPricePages, {{kFieldNamePrice, reindexer::KeyValueType::Int{}}, {kFieldNamePages, reindexer::KeyValueType::Int{}}}); addIndexFields(compositeIndexesNs, kCompositeFieldTitleName, @@ -127,12 +161,22 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNameColumnString, "-", "string", IndexOpts(), 0}, IndexDeclaration{kFieldNameColumnFullText, "text", "string", IndexOpts().SetConfig(R"xxx({"stemmers":[]})xxx"), 0}, IndexDeclaration{kFieldNameColumnStringNumeric, "-", "string", IndexOpts().SetCollateMode(CollateNumeric), 0}}); + addIndexFields(comparatorsNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnInt, {{kFieldNameColumnInt, reindexer::KeyValueType::Int{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnInt64, {{kFieldNameColumnInt64, reindexer::KeyValueType::Int64{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnDouble, {{kFieldNameColumnDouble, reindexer::KeyValueType::Double{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnString, {{kFieldNameColumnString, reindexer::KeyValueType::String{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnFullText, {{kFieldNameColumnFullText, reindexer::KeyValueType::String{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnStringNumeric, {{kFieldNameColumnStringNumeric, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(forcedSortOffsetNs); ASSERT_TRUE(err.ok()) << err.what(); DefineNamespaceDataset(forcedSortOffsetNs, {IndexDeclaration{kFieldNameId, "hash", "int", IndexOpts().PK(), 0}, IndexDeclaration{kFieldNameColumnHash, "hash", "int", IndexOpts(), 0}, IndexDeclaration{kFieldNameColumnTree, "tree", "int", IndexOpts(), 0}}); + addIndexFields(forcedSortOffsetNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(forcedSortOffsetNs, kFieldNameColumnHash, {{kFieldNameColumnHash, reindexer::KeyValueType::Int{}}}); + addIndexFields(forcedSortOffsetNs, kFieldNameColumnTree, {{kFieldNameColumnTree, reindexer::KeyValueType::Int{}}}); err = rt.reindexer->OpenNamespace(geomNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -142,12 +186,20 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNamePointLinearRTree, "rtree", "point", IndexOpts{}.RTreeType(IndexOpts::Linear), 0}, IndexDeclaration{kFieldNamePointGreeneRTree, "rtree", "point", IndexOpts{}.RTreeType(IndexOpts::Greene), 0}, IndexDeclaration{kFieldNamePointRStarRTree, "rtree", "point", IndexOpts{}.RTreeType(IndexOpts::RStar), 0}}); + addIndexFields(geomNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(geomNs, kFieldNamePointQuadraticRTree, {{kFieldNamePointQuadraticRTree, reindexer::KeyValueType::Double{}}}); + addIndexFields(geomNs, kFieldNamePointLinearRTree, {{kFieldNamePointLinearRTree, reindexer::KeyValueType::Double{}}}); + addIndexFields(geomNs, kFieldNamePointGreeneRTree, {{kFieldNamePointGreeneRTree, reindexer::KeyValueType::Double{}}}); + addIndexFields(geomNs, kFieldNamePointRStarRTree, {{kFieldNamePointRStarRTree, reindexer::KeyValueType::Double{}}}); err = rt.reindexer->OpenNamespace(btreeIdxOptNs); ASSERT_TRUE(err.ok()) << err.what(); DefineNamespaceDataset(btreeIdxOptNs, {IndexDeclaration{kFieldNameId, "tree", "int", IndexOpts().PK(), 0}, IndexDeclaration{kFieldNameStartTime, "tree", "int", IndexOpts(), 0}}); + addIndexFields(btreeIdxOptNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(btreeIdxOptNs, kFieldNameStartTime, {{kFieldNameStartTime, reindexer::KeyValueType::Int{}}}); initConditionsNs(); + initUUIDNs(); } void initConditionsNs(); @@ -155,6 +207,9 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { void CheckConditions(); enum class NullAllowed : bool { Yes = true, No = false }; void checkAllConditions(const std::string& fieldName, reindexer::KeyValueType fieldType, NullAllowed); + void initUUIDNs(); + void FillUUIDNs(); + void CheckUUIDQueries(); template void ExecuteAndVerify(const Query& query, T... args) { @@ -330,26 +385,26 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { const size_t id = i + lastId; bld.Put(kFieldNameId, id); - reindexer::Point point{randPoint(10)}; + reindexer::Point point{reindexer::randPoint(10)}; double arr[]{point.X(), point.Y()}; bld.Array(kFieldNamePointQuadraticRTree, reindexer::span{arr, 2}); - point = randPoint(10); + point = reindexer::randPoint(10); arr[0] = point.X(); arr[1] = point.Y(); bld.Array(kFieldNamePointLinearRTree, reindexer::span{arr, 2}); - point = randPoint(10); + point = reindexer::randPoint(10); arr[0] = point.X(); arr[1] = point.Y(); bld.Array(kFieldNamePointGreeneRTree, reindexer::span{arr, 2}); - point = randPoint(10); + point = reindexer::randPoint(10); arr[0] = point.X(); arr[1] = point.Y(); bld.Array(kFieldNamePointRStarRTree, reindexer::span{arr, 2}); - point = randPoint(10); + point = reindexer::randPoint(10); arr[0] = point.X(); arr[1] = point.Y(); bld.Array(kFieldNamePointNonIndex, reindexer::span{arr, 2}); @@ -515,8 +570,9 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { tr.Insert(std::move(item)); } QueryResults res; - rt.reindexer->CommitTransaction(tr, res); - const auto err = Commit(default_namespace); + auto err = rt.reindexer->CommitTransaction(tr, res); + ASSERT_TRUE(err.ok()) << err.what(); + err = Commit(default_namespace); ASSERT_TRUE(err.ok()) << err.what(); } @@ -599,6 +655,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { void CheckMergeQueriesWithAggregation(); void CheckGeomQueries() { + using reindexer::randPoint; + using reindexer::randBinDouble; for (size_t i = 0; i < 10; ++i) { // Checks that DWithin and sort by Distance work and verifies the result ExecuteAndVerify(Query(geomNs).DWithin(kFieldNamePointQuadraticRTree, randPoint(10), randBinDouble(0, 1))); @@ -699,8 +757,6 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { static const std::vector distincts = {"", kFieldNameYear, kFieldNameRate}; static const std::vector sortOrders = {true, false}; - static const std::string compositeIndexName(kFieldNameAge + compositePlus + kFieldNameGenre); - for (const bool sortOrder : sortOrders) { for (const auto& sortIdx : sortIdxs) { for (const std::string& distinct : distincts) { @@ -870,7 +926,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { // end of check substituteCompositIndexes ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNamePackages, CondEmpty, 0) + .Where(kFieldNamePackages, CondEmpty, {}) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); @@ -900,7 +956,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { ExecuteAndVerify(Query(default_namespace).Sort(kFieldNameGenre, true, {10, 20, 30})); ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNamePackages, CondAny, 0) + .Where(kFieldNamePackages, CondAny, {}) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); @@ -966,7 +1022,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Where(kFieldNameGenre, CondEq, 3) .Where(kFieldNamePackages, CondSet, RandIntVector(5, 10000, 50)) .Or() - .Where(kFieldNamePackages, CondEmpty, 0) + .Where(kFieldNamePackages, CondEmpty, {}) .Debug(LogTrace)); ExecuteAndVerify(Query(default_namespace) @@ -976,7 +1032,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Where(kFieldNameGenre, CondSet, {5, 1, 7}) .Where(kFieldNameYear, CondLt, 2010) .Or() - .Where(kFieldNamePackages, CondAny, 0) + .Where(kFieldNamePackages, CondAny, {}) .Where(kFieldNamePackages, CondSet, RandIntVector(5, 10000, 50)) .Debug(LogTrace)); @@ -1112,7 +1168,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .ReqTotal() .Distinct(distinct) .Sort(sortIdx, sortOrder) - .WhereComposite(compositeIndexName.c_str(), CondLe, {{Variant(27), Variant(10000)}})); + .WhereComposite(kCompositeFieldAgeGenre, CondLe, {{Variant(27), Variant(10000)}})); ExecuteAndVerify(Query(default_namespace) .ReqTotal() @@ -1125,7 +1181,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .ReqTotal() .Distinct(distinct) .Sort(sortIdx, sortOrder) - .WhereComposite(compositeIndexName.c_str(), CondEq, {{Variant(rand() % 10), Variant(rand() % 50)}})); + .WhereComposite(kCompositeFieldAgeGenre, CondEq, {{Variant(rand() % 10), Variant(rand() % 50)}})); ExecuteAndVerify(Query(default_namespace) .InnerJoin(kFieldNameYear, kFieldNameYear, CondEq, Query(joinNs)) @@ -1461,83 +1517,64 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .CloseBracket() .Distinct(distinct)); - for (CondType cond : {CondEq, CondSet, CondLt, CondLe, CondGt, CondGe}) { - const bool multyArgCond = cond == CondEq || cond == CondSet; - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, cond, randUuid()) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); - - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, cond, randStrUuid()) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); - - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, cond, - multyArgCond ? VariantArray::Create(randUuid(), randStrUuid(), randUuid(), - randStrUuid(), randUuid()) - : VariantArray::Create(randUuid())) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); - - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, cond, randUuid()) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); + for (CondType cond : {CondEq, CondSet, CondLt, CondLe, CondGt, CondGe, CondRange}) { + const auto argsCount = minMaxArgs(cond, 20); + if (argsCount.min <= 1 && argsCount.max >= 1) { + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuid, cond, randUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuid, cond, randStrUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuidArr, cond, randUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuidArr, cond, randStrUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + } ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, cond, randStrUuid()) + .Where(kFieldNameUuid, cond, randHeterogeneousUuidArray(argsCount.min, argsCount.max)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, cond, - multyArgCond ? VariantArray::Create(randUuid(), randStrUuid(), randUuid(), - randStrUuid(), randUuid(), randStrUuid()) - : VariantArray::Create(randUuid())) + .Where(kFieldNameUuidArr, cond, randHeterogeneousUuidArray(argsCount.min, argsCount.max)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); + std::vector compositeKeyValues; + VariantArray hetUuidArray = randHeterogeneousUuidArray(argsCount.min, argsCount.max); + compositeKeyValues.reserve(hetUuidArray.size()); + std::transform(std::make_move_iterator(hetUuidArray.begin()), std::make_move_iterator(hetUuidArray.end()), + std::back_inserter(compositeKeyValues), + [this](Variant&& uuid) { return VariantArray::Create(std::move(uuid), RandString()); }); ExecuteAndVerify(Query(default_namespace) - .WhereComposite(kFieldNameUuid + compositePlus + kFieldNameName, cond, - multyArgCond - ? std::vector{VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString())} - : std::vector{VariantArray::Create(randUuid(), RandString())}) + .WhereComposite(kCompositeFieldUuidName, cond, compositeKeyValues) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); } ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, CondRange, {nilUuid(), randUuid()}) + .Where(kFieldNameUuid, CondRange, randHeterogeneousUuidArray(2, 2)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, CondRange, {nilUuid(), randUuid()}) + .Where(kFieldNameUuidArr, CondRange, randHeterogeneousUuidArray(2, 2)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); ExecuteAndVerify(Query(default_namespace) - .WhereComposite(kFieldNameUuid + compositePlus + kFieldNameName, CondRange, + .WhereComposite(kCompositeFieldUuidName, CondRange, {VariantArray::Create(nilUuid(), RandString()), VariantArray::Create(randUuid(), RandString())}) .Distinct(distinct.c_str()) @@ -1841,6 +1878,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { // Checks that DSL queries works and compares the result with the result of corresponding C++ query void CheckDslQueries() { using namespace std::string_literals; + using reindexer::randPoint; + using reindexer::randBinDouble; // ---------- reindexer::Point point{randPoint(10)}; double distance = randBinDouble(0, 1); @@ -1876,6 +1915,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { void CheckSqlQueries() { using namespace std::string_literals; + using reindexer::randPoint; + using reindexer::randBinDouble; std::string sqlQuery = "SELECT ID, Year, Genre FROM test_namespace WHERE year > '2016' ORDER BY year DESC LIMIT 10000000"; const Query checkQuery1{Query(default_namespace, 0, 10000000).Where(kFieldNameYear, CondGt, 2016).Sort(kFieldNameYear, true)}; @@ -2178,7 +2219,13 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { const char* kFieldNamePages = "pages"; const char* kFieldNamePrice = "price"; const char* kFieldNameUuid = "uuid"; + const char* kFieldNameUuidSparse = "uuid_sparse"; const char* kFieldNameUuidArr = "uuid_arr"; + const char* kFieldNameUuidArrSparse = "uuid_arr_sparse"; + const char* kFieldNameUuidNotIndex = "uuid_not_index"; + const char* kFieldNameUuidNotIndex2 = "uuid_not_index_2"; + const char* kFieldNameUuidNotIndex3 = "uuid_not_index_3"; + const char* kFieldNameRndString = "rndString"; const char* kFieldNameBtreeIdsets = "btree_idsets"; const char* kFieldNamePointQuadraticRTree = "point_quadratic_rtree"; const char* kFieldNamePointLinearRTree = "point_linear_rtree"; @@ -2204,6 +2251,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { const std::string forcedSortOffsetNs = "forced_sort_offset_namespace"; const std::string nsWithObject = "namespace_with_object"; const std::string geomNs = "geom_namespace"; + const std::string uuidNs = "uuid_namespace"; const std::string btreeIdxOptNs = "btree_idx_opt_namespace"; const std::string conditionsNs = "conditions_namespace"; @@ -2221,6 +2269,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { static constexpr size_t forcedSortOffsetNsSize = 1000; static constexpr int forcedSortOffsetMaxValue = 1000; static constexpr size_t geomNsSize = 10000; + static constexpr size_t uuidNsSize = 10000; static constexpr int btreeIdxOptNsSize = 10000; size_t conditionsNsSize = 0; std::vector> forcedSortOffsetValues; diff --git a/cpp_src/gtests/tests/fixtures/queries_verifier.h b/cpp_src/gtests/tests/fixtures/queries_verifier.h index 861c05861..ed717fdca 100644 --- a/cpp_src/gtests/tests/fixtures/queries_verifier.h +++ b/cpp_src/gtests/tests/fixtures/queries_verifier.h @@ -26,6 +26,8 @@ class QueriesVerifier : public virtual ::testing::Test { std::string name; reindexer::KeyValueType type; }; + using IndexesData = std::unordered_map>; + void Verify(const reindexer::QueryResults& qr, const reindexer::Query& query, reindexer::Reindexer& rx) { std::unordered_set, PkHash> pks; std::unordered_map> distincts; @@ -40,14 +42,14 @@ class QueriesVerifier : public virtual ::testing::Test { ASSERT_TRUE(err.ok()) << err.what(); Verify(js.QueryResults(), js.JoinQuery(), rx); } - const auto& indexesFields = indexesFields_[query._namespace]; + const auto& indexesFields = indexesFields_[query.NsName()]; for (size_t i = 0; i < qr.Count(); ++i) { reindexer::Item itemr(qr[i].GetItem(false)); - auto pk = getPk(itemr, query._namespace); + auto pk = getPk(itemr, query.NsName()); EXPECT_TRUE(pks.insert(pk).second) << "Duplicated primary key: " + getPkString(pk); - InsertedItemsByPk& insertedItemsByPk = insertedItems_[query._namespace]; + InsertedItemsByPk& insertedItemsByPk = insertedItems_[query.NsName()]; auto itInsertedItem = insertedItemsByPk.find(pk); EXPECT_NE(itInsertedItem, insertedItemsByPk.end()) << "Item with such PK has not been inserted yet: " + getPkString(pk); if (itInsertedItem != insertedItemsByPk.end()) { @@ -142,14 +144,14 @@ class QueriesVerifier : public virtual ::testing::Test { // Check non found items, to not match conditions // If query has limit and offset, skip verification - if (query.start != 0 || query.count != reindexer::QueryEntry::kDefaultLimit) return; + if (query.HasOffset() || query.HasLimit()) return; // If query has distinct, skip verification for (const auto& agg : query.aggregations_) { if (agg.Type() == AggDistinct) return; } - for (auto& insertedItem : insertedItems_[query._namespace]) { + for (auto& insertedItem : insertedItems_[query.NsName()]) { if (pks.find(insertedItem.first) != pks.end()) continue; bool conditionsSatisfied = checkConditions(insertedItem.second, query.entries.cbegin(), query.entries.cend(), joinedSelectors, indexesFields); @@ -161,7 +163,7 @@ class QueriesVerifier : public virtual ::testing::Test { } auto aggResults = qr.GetAggregationResults(); - if (query.calcTotal != ModeNoTotal) { + if (query.CalcTotal() != ModeNoTotal) { // calcTotal from version 3.0.2 also return total count in aggregations, so we have remove it from here for // clean compare aggresults with aggregations aggResults.pop_back(); @@ -232,7 +234,7 @@ class QueriesVerifier : public virtual ::testing::Test { private: bool checkConditions(const reindexer::Item& item, reindexer::QueryEntries::const_iterator it, reindexer::QueryEntries::const_iterator to, const std::vector& joinedSelectors, - const std::unordered_map>& indexesFields) { + const IndexesData& indexesFields) { bool result = true; for (; it != to; ++it) { OpType op = it->operation; @@ -247,7 +249,7 @@ class QueriesVerifier : public virtual ::testing::Test { return checkConditions(item, it.cbegin(), it.cend(), joinedSelectors, indexesFields); }, [&](const reindexer::QueryEntry& qe) { - if ((op == OpOr && result) || qe.distinct) { + if ((op == OpOr && result) || qe.Distinct()) { skip = true; return false; } @@ -287,8 +289,7 @@ class QueriesVerifier : public virtual ::testing::Test { return result; } - static std::string getFieldName(const std::string& indexName, - const std::unordered_map>& indexesFields) { + static std::string getFieldName(const std::string& indexName, const IndexesData& indexesFields) { if (const auto it = indexesFields.find(indexName); it == indexesFields.end()) { return indexName; } else { @@ -300,15 +301,15 @@ class QueriesVerifier : public virtual ::testing::Test { static bool checkDistincts(reindexer::Item& item, const reindexer::Query& qr, std::unordered_map>& distincts, - const std::unordered_map>& indexesFields) { + const IndexesData& indexesFields) { bool result = true; // check only on root level for (auto it = qr.entries.cbegin(); it != qr.entries.cend(); ++it) { if (!it->HoldsOrReferTo()) continue; const reindexer::QueryEntry& qentry = it->Value(); - if (!qentry.distinct) continue; + if (!qentry.Distinct()) continue; - const std::string fieldName = getFieldName(qentry.index, indexesFields); + const std::string fieldName = getFieldName(qentry.FieldName(), indexesFields); reindexer::VariantArray fieldValue = item[fieldName]; EXPECT_EQ(fieldValue.size(), 1) << "Distinct field's size cannot be > 1"; if (fieldValue.empty()) return false; @@ -316,30 +317,30 @@ class QueriesVerifier : public virtual ::testing::Test { std::unordered_set& values = distincts[fieldName]; reindexer::Variant keyValue(fieldValue[0]); bool inserted = values.insert(keyValue.As()).second; - EXPECT_TRUE(inserted) << "Duplicate distinct item for index: " << keyValue.As() << ", " << qentry.idxNo; + EXPECT_TRUE(inserted) << "Duplicate distinct item for index: " << keyValue.As() << ", " << qentry.FieldName() + << " (" << qentry.IndexNo() << ')'; result &= inserted; } return result; } - bool checkCondition(const reindexer::Item& item, const JoinedSelectorMock& joinedSelector, - const std::unordered_map>& leftIndexesFields, - const std::unordered_map>& rightIndexesFields) { + bool checkCondition(const reindexer::Item& item, const JoinedSelectorMock& joinedSelector, const IndexesData& leftIndexesFields, + const IndexesData& rightIndexesFields) { for (auto it : joinedSelector.QueryResults()) { const reindexer::Item& rightItem = it.GetItem(false); bool result = true; const auto& joinEntries{joinedSelector.JoinQuery().joinEntries_}; assertrx(!joinEntries.empty()); - assertrx(joinEntries[0].op_ != OpOr); + assertrx(joinEntries[0].Operation() != OpOr); for (const auto& je : joinEntries) { - if (je.op_ == OpOr) { + if (je.Operation() == OpOr) { if (result) continue; } else if (!result) { break; } - const bool curResult = - checkOnCondition(item, rightItem, je.index_, je.joinIndex_, je.condition_, leftIndexesFields, rightIndexesFields); - switch (je.op_) { + const bool curResult = checkOnCondition(item, rightItem, je.LeftFieldName(), je.RightFieldName(), je.Condition(), + leftIndexesFields, rightIndexesFields); + switch (je.Operation()) { case OpAnd: result = curResult; break; @@ -359,9 +360,8 @@ class QueriesVerifier : public virtual ::testing::Test { } bool checkOnCondition(const reindexer::Item& leftItem, const reindexer::Item& rightItem, const std::string& leftIndexName, - const std::string& rightIndexName, CondType cond, - const std::unordered_map>& leftIndexesFields, - const std::unordered_map>& rightIndexesFields) { + const std::string& rightIndexName, CondType cond, const IndexesData& leftIndexesFields, + const IndexesData& rightIndexesFields) { const CollateOpts& collate = indexesCollates[leftIndexName]; const std::string leftFieldName = getFieldName(leftIndexName, leftIndexesFields); const std::string rightFieldName = getFieldName(rightIndexName, rightIndexesFields); @@ -375,21 +375,20 @@ class QueriesVerifier : public virtual ::testing::Test { return false; } - bool checkCondition(const reindexer::Item& item, const reindexer::QueryEntry& qentry, - const std::unordered_map>& indexesFields) { + bool checkCondition(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const IndexesData& indexesFields) { EXPECT_GT(item.NumFields(), 0); - if (isGeomConditions(qentry.condition)) { + if (isGeomConditions(qentry.Condition())) { return checkGeomConditions(item, qentry, indexesFields); } - const CollateOpts& collate = indexesCollates[qentry.index]; + const CollateOpts& collate = indexesCollates[qentry.FieldName()]; - if (isIndexComposite(item, qentry)) { - return checkCompositeValues(item, qentry, collate, indexesFields); + if (isIndexComposite(qentry.FieldName(), indexesFields)) { + return checkCompositeCondition(item, qentry, collate, indexesFields); } else { std::string fieldName; reindexer::KeyValueType fieldType = reindexer::KeyValueType::Undefined{}; - if (const auto it = indexesFields.find(qentry.index); it == indexesFields.end()) { - fieldName = qentry.index; + if (const auto it = indexesFields.find(qentry.FieldName()); it == indexesFields.end()) { + fieldName = qentry.FieldName(); } else { EXPECT_EQ(it->second.size(), 1); assertrx(!it->second.empty()); @@ -397,13 +396,13 @@ class QueriesVerifier : public virtual ::testing::Test { fieldType = it->second[0].type; } reindexer::VariantArray fieldValues = item[fieldName]; - switch (qentry.condition) { + switch (qentry.Condition()) { case CondEmpty: return fieldValues.size() == 0; case CondAny: return fieldValues.size() > 0; case CondAllSet: - return checkAllSet(fieldValues, qentry.values, collate, fieldType); + return checkAllSet(fieldValues, qentry.Values(), collate, fieldType); case CondEq: case CondLt: case CondLe: @@ -414,7 +413,8 @@ class QueriesVerifier : public virtual ::testing::Test { case CondLike: case CondDWithin: for (const reindexer::Variant& fieldValue : fieldValues) { - if (compareValue(fieldValue, qentry.condition, qentry.values, collate, fieldType)) return true; + if (compareValue(fieldValue, qentry.Condition(), qentry.Values(), collate, fieldType)) + return true; } } } @@ -424,16 +424,15 @@ class QueriesVerifier : public virtual ::testing::Test { static bool isGeomConditions(CondType cond) noexcept { return cond == CondType::CondDWithin; } - static bool checkGeomConditions(const reindexer::Item& item, const reindexer::QueryEntry& qentry, - const std::unordered_map>& indexesFields) { - assertrx(qentry.values.size() == 2); - const reindexer::VariantArray coordinates = item[getFieldName(qentry.index, indexesFields)]; + static bool checkGeomConditions(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const IndexesData& indexesFields) { + assertrx(qentry.Values().size() == 2); + const reindexer::VariantArray coordinates = item[getFieldName(qentry.FieldName(), indexesFields)]; if (coordinates.empty()) return false; assertrx(coordinates.size() == 2); const double x = coordinates[0].As(); const double y = coordinates[1].As(); - if (qentry.condition == CondDWithin) { - return DWithin(reindexer::Point{x, y}, qentry.values[0].As(), qentry.values[1].As()); + if (qentry.Condition() == CondDWithin) { + return DWithin(reindexer::Point{x, y}, qentry.Values()[0].As(), qentry.Values()[1].As()); } else { assertrx(0); abort(); @@ -448,25 +447,24 @@ class QueriesVerifier : public virtual ::testing::Test { return kvalues; } - static std::vector getCompositeFields(const std::string& indexName, - const std::unordered_map>& indexesFields) { + static const std::vector& getCompositeFields(const std::string& indexName, const IndexesData& indexesFields) { const auto it = indexesFields.find(indexName); assert(it != indexesFields.end()); return it->second; } - static bool checkCompositeValues(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const CollateOpts& opts, - const std::unordered_map>& indexesFields) { - const auto fields = getCompositeFields(qentry.index, indexesFields); + static bool checkCompositeCondition(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const CollateOpts& opts, + const IndexesData& indexesFields) { + const auto fields = getCompositeFields(qentry.FieldName(), indexesFields); const reindexer::VariantArray& indexesValues = getValues(item, fields); - const reindexer::VariantArray& keyValues = qentry.values; + const reindexer::VariantArray& keyValues = qentry.Values(); - switch (qentry.condition) { + switch (qentry.Condition()) { case CondEmpty: - return indexesValues.size() == 0; + return indexesValues.empty(); case CondAny: - return indexesValues.size() > 0; + return !indexesValues.empty(); case CondGe: assert(!keyValues.empty()); return compareCompositeValues(indexesValues, keyValues[0], opts) >= 0; @@ -490,10 +488,10 @@ class QueriesVerifier : public virtual ::testing::Test { } return false; case CondAllSet: - for (const reindexer::Variant& kv : indexesValues) { + for (const reindexer::Variant& kv : keyValues) { if (compareCompositeValues(indexesValues, kv, opts) != 0) return false; } - return !indexesValues.empty(); + return !keyValues.empty(); case CondLike: case CondDWithin: default: @@ -640,51 +638,42 @@ class QueriesVerifier : public virtual ::testing::Test { } bool checkCompositeCondition(const reindexer::Item& item, const reindexer::BetweenFieldsQueryEntry& qentry, - const std::unordered_map>& indexesFields) { - const auto firstFields = getCompositeFields(qentry.firstIndex, indexesFields); - const auto secondFields = getCompositeFields(qentry.secondIndex, indexesFields); + const IndexesData& indexesFields) { + const auto& firstFields = getCompositeFields(qentry.LeftFieldName(), indexesFields); + const auto& secondFields = getCompositeFields(qentry.RightFieldName(), indexesFields); assertrx(firstFields.size() == secondFields.size()); - reindexer::BetweenFieldsQueryEntry qe{qentry}; for (size_t i = 0; i < firstFields.size(); ++i) { - qe.firstIndex = firstFields[i].name; - qe.secondIndex = secondFields[i].name; - if (!checkCondition(item, qe, indexesFields)) return false; + if (!checkCondition(item, + reindexer::BetweenFieldsQueryEntry{std::string{firstFields[i].name}, qentry.Condition(), + std::string{secondFields[i].name}}, + indexesFields)) + return false; } return !firstFields.empty(); } - static bool isIndexComposite(const reindexer::BetweenFieldsQueryEntry& qe, - const std::unordered_map>& indexesFields) { - if (qe.firstIndex.find('+') != std::string::npos || qe.secondIndex.find('+') != std::string::npos) return true; - if (const auto it = indexesFields.find(qe.firstIndex); it != indexesFields.end() && it->second.size() > 1) return true; - if (const auto it = indexesFields.find(qe.secondIndex); it != indexesFields.end() && it->second.size() > 1) return true; - return false; + static bool isIndexComposite(const std::string& indexName, const IndexesData& indexesFields) { + const auto it = indexesFields.find(indexName); + return it != indexesFields.end() && it->second.size() > 1; } - static bool isIndexComposite(const reindexer::Item& item, const reindexer::QueryEntry& qentry) { - if (qentry.values.empty()) return false; - if (qentry.idxNo < 0) { - return qentry.values.size() && (qentry.values[0].Type().Is() || - qentry.values[0].Type().Is()); - } - const auto indexType = item.GetIndexType(qentry.idxNo); - return indexType.Is() || indexType.Is(); + static bool isIndexComposite(const reindexer::BetweenFieldsQueryEntry& qe, const IndexesData& indexesFields) { + return isIndexComposite(qe.LeftFieldName(), indexesFields) || isIndexComposite(qe.RightFieldName(), indexesFields); } - bool checkCondition(const reindexer::Item& item, const reindexer::BetweenFieldsQueryEntry& qentry, - const std::unordered_map>& indexesFields) { + bool checkCondition(const reindexer::Item& item, const reindexer::BetweenFieldsQueryEntry& qentry, const IndexesData& indexesFields) { EXPECT_GT(item.NumFields(), 0); assertrx(!isGeomConditions(qentry.Condition())); - const CollateOpts& collate = indexesCollates[qentry.firstIndex]; + const CollateOpts& collate = indexesCollates[qentry.LeftFieldName()]; if (isIndexComposite(qentry, indexesFields)) { return checkCompositeCondition(item, qentry, indexesFields); } - const std::string firstField = getFieldName(qentry.firstIndex, indexesFields); - const std::string secondField = getFieldName(qentry.secondIndex, indexesFields); + const std::string firstField = getFieldName(qentry.LeftFieldName(), indexesFields); + const std::string secondField = getFieldName(qentry.RightFieldName(), indexesFields); reindexer::VariantArray lValues = item[firstField]; reindexer::VariantArray rValues = item[secondField]; switch (qentry.Condition()) { @@ -811,8 +800,8 @@ class QueriesVerifier : public virtual ::testing::Test { std::vector result; result.reserve(query.joinQueries_.size()); for (auto jq : query.joinQueries_) { - jq.count = reindexer::QueryEntry::kDefaultLimit; - jq.start = reindexer::QueryEntry::kDefaultOffset; + jq.Limit(reindexer::QueryEntry::kDefaultLimit); + jq.Offset(reindexer::QueryEntry::kDefaultOffset); jq.sortingEntries_.clear(); jq.forcedSortOrder_.clear(); result.emplace_back(InnerJoin, std::move(jq)); @@ -915,5 +904,5 @@ class QueriesVerifier : public virtual ::testing::Test { } std::unordered_map> ns2pk_; - std::unordered_map>> indexesFields_; + std::unordered_map indexesFields_; }; diff --git a/cpp_src/gtests/tests/fixtures/reindexer_api.h b/cpp_src/gtests/tests/fixtures/reindexer_api.h index 0cc0c4c84..194c32c72 100644 --- a/cpp_src/gtests/tests/fixtures/reindexer_api.h +++ b/cpp_src/gtests/tests/fixtures/reindexer_api.h @@ -6,16 +6,12 @@ #include #include -#include "core/keyvalue/key_string.h" #include "core/keyvalue/variant.h" #include "core/query/query.h" #include "core/reindexer.h" #include "reindexertestapi.h" #include "servercontrol.h" #include "tools/errors.h" -#include "tools/serializer.h" -#include "tools/stringstools.h" -#include "vendor/utf8cpp/utf8.h" using reindexer::Error; using reindexer::Item; diff --git a/cpp_src/gtests/tests/fixtures/runtime_indexes_api.h b/cpp_src/gtests/tests/fixtures/runtime_indexes_api.h index f7358208f..bffeb84bc 100644 --- a/cpp_src/gtests/tests/fixtures/runtime_indexes_api.h +++ b/cpp_src/gtests/tests/fixtures/runtime_indexes_api.h @@ -2,7 +2,7 @@ #include "gtests/tools.h" #include "reindexer_api.h" -#include "tools/random.h" +#include "tools/randompoint.h" class RuntimeIndexesApi : public ReindexerApi { public: @@ -32,6 +32,7 @@ class RuntimeIndexesApi : public ReindexerApi { protected: void FillNamespaces(size_t since, size_t till) { + using reindexer::randPoint; for (size_t i = since; i < till; ++i) { int id = static_cast(i); @@ -245,7 +246,7 @@ class RuntimeIndexesApi : public ReindexerApi { std::string indexName = getRuntimeQPointIndexName(indexNumber); for (size_t i = 0; i < 10; ++i) { Item item = NewItem(geom_namespace); - item[indexName] = randPoint(10); + item[indexName] = reindexer::randPoint(10); Upsert(geom_namespace, item); } auto err = Commit(geom_namespace); @@ -256,7 +257,7 @@ class RuntimeIndexesApi : public ReindexerApi { std::string indexName = getRuntimeLPointIndexName(indexNumber); for (size_t i = 0; i < 10; ++i) { Item item = NewItem(geom_namespace); - item[indexName] = randPoint(10); + item[indexName] = reindexer::randPoint(10); Upsert(geom_namespace, item); } auto err = Commit(geom_namespace); @@ -267,7 +268,7 @@ class RuntimeIndexesApi : public ReindexerApi { std::string indexName = getRuntimeGPointIndexName(indexNumber); for (size_t i = 0; i < 10; ++i) { Item item = NewItem(geom_namespace); - item[indexName] = randPoint(10); + item[indexName] = reindexer::randPoint(10); Upsert(geom_namespace, item); } auto err = Commit(geom_namespace); @@ -278,7 +279,7 @@ class RuntimeIndexesApi : public ReindexerApi { std::string indexName = getRuntimeSPointIndexName(indexNumber); for (size_t i = 0; i < 10; ++i) { Item item = NewItem(geom_namespace); - item[indexName] = randPoint(10); + item[indexName] = reindexer::randPoint(10); Upsert(geom_namespace, item); } auto err = Commit(geom_namespace); diff --git a/cpp_src/gtests/tests/fixtures/servercontrol.cc b/cpp_src/gtests/tests/fixtures/servercontrol.cc index 59cfe9e97..bcbfc80f2 100644 --- a/cpp_src/gtests/tests/fixtures/servercontrol.cc +++ b/cpp_src/gtests/tests/fixtures/servercontrol.cc @@ -124,9 +124,31 @@ void ServerControl::Interface::WriteServerConfig(const std::string& configYaml) void ServerControl::Interface::SetWALSize(int64_t size, std::string_view nsName) { setNamespaceConfigItem(nsName, "wal_size", size); } +void ServerControl::Interface::SetTxAlwaysCopySize(int64_t size, std::string_view nsName) { + setNamespaceConfigItem(nsName, "tx_size_to_always_copy", size); +} + void ServerControl::Interface::SetOptmizationSortWorkers(size_t cnt, std::string_view nsName) { setNamespaceConfigItem(nsName, "optimization_sort_workers", cnt); } + +void ServerControl::Interface::EnableAllProfilings() { + constexpr std::string_view kJsonCfgProfiling = R"json({ + "type":"profiling", + "profiling":{ + "queriesperfstats":true, + "queries_threshold_us":0, + "perfstats":true, + "memstats":true + } + })json"; + auto item = api.reindexer->NewItem(kConfigNs); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + auto err = item.FromJSON(kJsonCfgProfiling); + ASSERT_TRUE(err.ok()) << err.what(); + err = api.reindexer->Upsert(kConfigNs, item); + ASSERT_TRUE(err.ok()) << err.what(); +} void ServerControl::Interface::Init() { stopped_ = false; YAML::Node y; diff --git a/cpp_src/gtests/tests/fixtures/servercontrol.h b/cpp_src/gtests/tests/fixtures/servercontrol.h index b268f0b26..86a4154ab 100644 --- a/cpp_src/gtests/tests/fixtures/servercontrol.h +++ b/cpp_src/gtests/tests/fixtures/servercontrol.h @@ -112,8 +112,9 @@ class ServerControl { void WriteServerConfig(const std::string& configYaml); // set server's WAL size void SetWALSize(int64_t size, std::string_view nsName); - + void SetTxAlwaysCopySize(int64_t size, std::string_view nsName); void SetOptmizationSortWorkers(size_t cnt, std::string_view nsName); + void EnableAllProfilings(); reindexer_server::Server srv; #ifndef _WIN32 diff --git a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h index 4feca8291..c32ec7caf 100644 --- a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h +++ b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h @@ -86,7 +86,10 @@ class DISABLED_StorageLazyLoadApi : public ReindexerApi { ASSERT_TRUE(err.ok()) << err.what(); } - void dropNs() { rt.reindexer->DropNamespace(default_namespace); } + void dropNs() { + const auto err = rt.reindexer->DropNamespace(default_namespace); + ASSERT_TRUE(err.ok()) << err.what(); + } int64_t getItemsCount(bool& storageLoaded) { QueryResults qr; diff --git a/cpp_src/gtests/tests/fixtures/systemhelpers.cc b/cpp_src/gtests/tests/fixtures/systemhelpers.cc index dbe3d3ebd..fae4110cb 100644 --- a/cpp_src/gtests/tests/fixtures/systemhelpers.cc +++ b/cpp_src/gtests/tests/fixtures/systemhelpers.cc @@ -34,7 +34,7 @@ pid_t StartProcess(const std::string& program, const std::vector& p if (isMainThread) { // prctl sends signal on thread termination, so this call may lead to unexpected process termination int r = prctl(PR_SET_PDEATHSIG, SIGTERM); if (r == -1) { - perror("prctl error\n"); + perror("prctl error"); exit(1); } } @@ -44,7 +44,7 @@ pid_t StartProcess(const std::string& program, const std::vector& p } int ret = execv(program.c_str(), ¶msPointers[0]); if (ret) { - perror("exec error\n"); + perror("exec error"); exit(1); } } diff --git a/cpp_src/gtests/tests/fixtures/ttl_index_api.h b/cpp_src/gtests/tests/fixtures/ttl_index_api.h index 9e6768fef..5026ed6e0 100644 --- a/cpp_src/gtests/tests/fixtures/ttl_index_api.h +++ b/cpp_src/gtests/tests/fixtures/ttl_index_api.h @@ -11,7 +11,8 @@ class TtlIndexApi : public ReindexerApi { DefineNamespaceDataset(default_namespace, {IndexDeclaration{kFieldId, "hash", "int", IndexOpts().PK(), 0}, IndexDeclaration{kFieldData, "tree", "int", IndexOpts().Array(), 0}, IndexDeclaration{kFieldData, "tree", "int", IndexOpts().Array(), 0}}); - rt.reindexer->AddIndex(default_namespace, reindexer::IndexDef(kFieldDate, {kFieldDate}, "ttl", "int64", IndexOpts(), 1)); + err = rt.reindexer->AddIndex(default_namespace, reindexer::IndexDef(kFieldDate, {kFieldDate}, "ttl", "int64", IndexOpts(), 1)); + ASSERT_TRUE(err.ok()) << err.what(); AddDataToNs(3000); } diff --git a/cpp_src/gtests/tests/fuzzing/fuzzing.cc b/cpp_src/gtests/tests/fuzzing/fuzzing.cc index 5491b6c69..fb71502d8 100644 --- a/cpp_src/gtests/tests/fuzzing/fuzzing.cc +++ b/cpp_src/gtests/tests/fuzzing/fuzzing.cc @@ -1,4 +1,6 @@ #include "fuzzing/fuzzing.h" +#include "args/args.hpp" +#include "fuzzing/index.h" #include "fuzzing/ns.h" #include "fuzzing/query_generator.h" @@ -6,23 +8,25 @@ TEST_F(Fuzzing, BaseTest) { try { const fuzzing::RandomGenerator::ErrFactorType errorFactor{0, 1}; reindexer::WrSerializer ser; - std::unordered_set generatedNames; - fuzzing::RandomGenerator rnd(std::cout, errorFactor); - std::vector namespaces_; + fuzzing::RandomGenerator rnd(errorFactor); + std::vector namespaces; const size_t N = 1; for (size_t i = 0; i < N; ++i) { - namespaces_.emplace_back(rnd.NsName(generatedNames), std::cout, errorFactor); - fuzzing::Ns& ns = namespaces_.back(); + namespaces.emplace_back(rnd.GenerateNsName(), errorFactor); + fuzzing::Ns& ns = namespaces.back(); auto err = rx_.OpenNamespace(ns.GetName()); EXPECT_TRUE(err.ok()) << err.what(); - if (!err.ok()) continue; + if (!err.ok()) { + continue; + } auto& indexes = ns.GetIndexes(); - for (size_t i = 0; i < indexes.size();) { - const auto idxDef = indexes[i].IndexDef(ns.GetRandomGenerator(), ns.GetScheme()); + for (size_t j = 0; j < indexes.size();) { + const fuzzing::Index& idx = indexes[j]; + const auto idxDef = idx.IndexDef(ns.GetRandomGenerator(), ns.GetScheme(), indexes); err = rx_.AddIndex(ns.GetName(), idxDef); EXPECT_TRUE(err.ok()) << err.what(); if (err.ok()) { - ns.AddIndex(indexes[i], !idxDef.opts_.IsDense() && idxDef.opts_.IsSparse() && !idxDef.opts_.IsPK()); + ns.AddIndexToScheme(idx, j); // TODO move to fuzzing::Ns std::vector fields; std::visit(reindexer::overloaded{ [&](const fuzzing::Index::Child& c) { @@ -35,23 +39,22 @@ TEST_F(Fuzzing, BaseTest) { ToKeyValueType(ns.GetScheme().GetFieldType(child.fieldPath))}); } }}, - indexes[i].content); - if (indexes[i].isPk) { + idx.Content()); + if (idx.IsPk()) { setPkFields(ns.GetName(), fields); } - - addIndexFields(ns.GetName(), indexes[i].name, std::move(fields)); - ++i; + addIndexFields(ns.GetName(), idx.Name(), std::move(fields)); + ++j; } else { - indexes.erase(indexes.begin() + i); + indexes.erase(indexes.begin() + j); } } - for (size_t i = 0, s = ns.GetRandomGenerator().RndItemsCount(); i < s; ++i) { + for (size_t j = 0, s = ns.GetRandomGenerator().RndItemsCount(); j < s; ++j) { auto item = rx_.NewItem(ns.GetName()); err = item.Status(); EXPECT_TRUE(err.ok()) << err.what(); if (!err.ok()) continue; - ns.NewItem(ser); + ns.NewItem(ser); // TODO not json err = item.FromJSON(ser.Slice()); EXPECT_TRUE(err.ok()) << err.what() << std::endl << "size: " << ser.Slice().size() << std::endl << ser.Slice(); if (!err.ok()) continue; @@ -61,26 +64,26 @@ TEST_F(Fuzzing, BaseTest) { enum Op : uint8_t { Insert, Upsert, Update, Delete, END = Delete }; switch (rnd.RndWhich()) { case Insert: - err = rx_.Insert(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Insert(rnd.NsName(ns.GetName()), item); if (err.ok() && item.GetID() != -1) { saveItem(std::move(item), ns.GetName()); } break; case Upsert: - err = rx_.Upsert(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Upsert(rnd.NsName(ns.GetName()), item); if (err.ok()) { saveItem(std::move(item), ns.GetName()); } break; case Update: - err = rx_.Update(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Update(rnd.NsName(ns.GetName()), item); if (err.ok() && item.GetID() != -1) { saveItem(std::move(item), ns.GetName()); } break; case Delete: { const auto id = item.GetID(); - err = rx_.Delete(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Delete(rnd.NsName(ns.GetName()), item); if (err.ok() && item.GetID() != id) { deleteItem(item, ns.GetName()); } @@ -94,7 +97,7 @@ TEST_F(Fuzzing, BaseTest) { err = rx_.Select(reindexer::Query(ns.GetName()).ReqTotal(), qr); EXPECT_TRUE(err.ok()) << err.what(); } - fuzzing::QueryGenerator queryGenerator{namespaces_, std::cout, errorFactor}; + fuzzing::QueryGenerator queryGenerator{namespaces, errorFactor}; for (size_t i = 0; i < 100; ++i) { auto query = queryGenerator(); reindexer::QueryResults qr; @@ -113,5 +116,40 @@ TEST_F(Fuzzing, BaseTest) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); + + args::ArgumentParser parser("Reindexer fuzzing tests"); + args::HelpFlag help(parser, "help", "show this message", {'h', "help"}); + args::Group progOptions("options"); + args::ValueFlag dbDsn(progOptions, "DSN", + "DSN to 'reindexer'. Can be 'cproto://:/' or 'builtin://'", {'d', "dsn"}, + args::Options::Single | args::Options::Global); + args::ValueFlag output(progOptions, "FILENAME", "A file for saving initial states of random engines", {'s', "save"}, + args::Options::Single | args::Options::Global); + args::ValueFlag input(progOptions, "FILENAME", "A file for initial states of random engines recovery", {'r', "repeat"}, + args::Options::Single | args::Options::Global); + args::GlobalOptions globals(parser, progOptions); + try { + parser.ParseCLI(argc, argv); + } catch (const args::Help&) { + std::cout << parser.Help() << std::endl; + return 1; + } catch (const args::Error& e) { + std::cerr << "ERROR: " << e.what() << std::endl; + std::cout << parser.Help() << std::endl; + return 1; + } + std::string out = args::get(output); + if (!out.empty()) { + fuzzing::RandomGenerator::SetOut(std::move(out)); + } + const std::string in = args::get(input); + if (!in.empty()) { + fuzzing::RandomGenerator::SetIn(in); + } + std::string dsn = args::get(dbDsn); + if (!dsn.empty()) { + Fuzzing::SetDsn(std::move(dsn)); + } + return RUN_ALL_TESTS(); } diff --git a/cpp_src/gtests/tests/mocks/rpcserver_fake.cc b/cpp_src/gtests/tests/mocks/rpcserver_fake.cc index e7611f3f6..abb9d3511 100644 --- a/cpp_src/gtests/tests/mocks/rpcserver_fake.cc +++ b/cpp_src/gtests/tests/mocks/rpcserver_fake.cc @@ -124,8 +124,9 @@ bool RPCServerFake::Start(const std::string &addr, ev::dynamic_loop &loop, Error dispatcher_.Middleware(this, &RPCServerFake::CheckAuth); - listener_.reset(new Listener(loop, cproto::ServerConnection::NewFactory(dispatcher_, false, 1024 * 1024 * 1024))); - return listener_->Bind(addr); + listener_ = + std::make_unique>(loop, cproto::ServerConnection::NewFactory(dispatcher_, false, 1024 * 1024 * 1024)); + return listener_->Bind(addr, socket_domain::tcp); } RPCServerStatus RPCServerFake::Status() const { return state_; } diff --git a/cpp_src/gtests/tests/mocks/rpcserver_fake.h b/cpp_src/gtests/tests/mocks/rpcserver_fake.h index a039e1509..e99ee9585 100644 --- a/cpp_src/gtests/tests/mocks/rpcserver_fake.h +++ b/cpp_src/gtests/tests/mocks/rpcserver_fake.h @@ -19,7 +19,7 @@ struct RPCServerConfig { enum RPCServerStatus { Init, Connected, Stopped }; -struct RPCClientData : public cproto::ClientData { +struct RPCClientData final : public cproto::ClientData { AuthContext auth; int connID; }; diff --git a/cpp_src/gtests/tests/unit/composite_indexes_api.h b/cpp_src/gtests/tests/unit/composite_indexes_api.h index a555bb162..e78f25c55 100644 --- a/cpp_src/gtests/tests/unit/composite_indexes_api.h +++ b/cpp_src/gtests/tests/unit/composite_indexes_api.h @@ -91,16 +91,24 @@ class CompositeIndexesApi : public ReindexerApi { QueryResults qr; auto err = rt.reindexer->Select(query, qr); EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); QueryResults qrSql; auto sqlQuery = query.GetSQL(); err = rt.reindexer->Select(query.GetSQL(), qrSql); EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); EXPECT_EQ(qr.Count(), qrSql.Count()) << "SQL: " << sqlQuery; for (auto it = qr.begin(), itSql = qrSql.begin(); it != qr.end() && itSql != qrSql.end(); ++it, ++itSql) { + EXPECT_TRUE(it.Status().ok()) << it.Status().what(); + assert(it.Status().ok()); reindexer::WrSerializer ser, serSql; - it.GetCJSON(ser); - itSql.GetCJSON(serSql); + err = it.GetCJSON(ser); + EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); + err = itSql.GetCJSON(serSql); + EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); EXPECT_EQ(ser.Slice(), serSql.Slice()) << "SQL: " << sqlQuery; } return qr; diff --git a/cpp_src/gtests/tests/unit/composite_indexes_test.cc b/cpp_src/gtests/tests/unit/composite_indexes_test.cc index b6ab2b70c..c5579d1b5 100644 --- a/cpp_src/gtests/tests/unit/composite_indexes_test.cc +++ b/cpp_src/gtests/tests/unit/composite_indexes_test.cc @@ -46,11 +46,13 @@ TEST_F(CompositeIndexesApi, AddIndexWithExistingCompositeIndex) { static void selectAll(reindexer::Reindexer* reindexer, const std::string& ns) { QueryResults qr; Error err = reindexer->Select(Query(ns, 0, 1000, ModeAccurateTotal), qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); for (auto it : qr) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer wrser; - it.GetJSON(wrser, false); + err = it.GetJSON(wrser, false); + ASSERT_TRUE(err.ok()) << err.what(); } } @@ -64,7 +66,7 @@ TEST_F(CompositeIndexesApi, DropTest2) { for (int i = 0; i < 1000; ++i) { Item item = NewItem(test_ns); - EXPECT_TRUE(!!item); + EXPECT_FALSE(!item); EXPECT_TRUE(item.Status().ok()) << item.Status().what(); item["id"] = i + 1; @@ -118,7 +120,7 @@ TEST_F(CompositeIndexesApi, CompositeIndexesSelectTest) { auto qr = execAndCompareQuery( Query(default_namespace).WhereComposite(compositeIndexName.c_str(), CondEq, {{Variant(priceValue), Variant(pagesValue)}})); - EXPECT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); Item pricePageRow = qr.begin().GetItem(false); Variant selectedPrice = pricePageRow[kFieldNamePrice]; @@ -129,8 +131,8 @@ TEST_F(CompositeIndexesApi, CompositeIndexesSelectTest) { Item titleNameRow = qr.begin().GetItem(false); Variant selectedTitle = titleNameRow[kFieldNameTitle]; Variant selectedName = titleNameRow[kFieldNameName]; - EXPECT_TRUE(static_cast(selectedTitle)->compare(std::string(titleValue)) == 0); - EXPECT_TRUE(static_cast(selectedName)->compare(std::string(nameValue)) == 0); + EXPECT_EQ(static_cast(selectedTitle)->compare(std::string(titleValue)), 0); + EXPECT_EQ(static_cast(selectedName)->compare(std::string(nameValue)), 0); execAndCompareQuery(Query(default_namespace).WhereComposite(compositeIndexName, CondLt, {{Variant(priceValue), Variant(pagesValue)}})); execAndCompareQuery(Query(default_namespace).WhereComposite(compositeIndexName, CondLe, {{Variant(priceValue), Variant(pagesValue)}})); diff --git a/cpp_src/gtests/tests/unit/dsl_parser_test.cc b/cpp_src/gtests/tests/unit/dsl_parser_test.cc index 0daee4f86..2fb4bb4c7 100644 --- a/cpp_src/gtests/tests/unit/dsl_parser_test.cc +++ b/cpp_src/gtests/tests/unit/dsl_parser_test.cc @@ -1,17 +1,20 @@ #include "join_selects_api.h" +static void checkQueryDslParse(const reindexer::Query& q) { + const std::string dsl = q.GetJSON(); + Query parsedQuery; + Error err = parsedQuery.FromJSON(dsl); + ASSERT_TRUE(err.ok()) << err.what() << "\nDSL:\n" << dsl; + ASSERT_EQ(q, parsedQuery) << "DSL:\n" << dsl << "\nOriginal query:\n" << q.GetSQL() << "\nParsed query:\n" << parsedQuery.GetSQL(); +} + TEST_F(JoinSelectsApi, JoinsDSLTest) { Query queryGenres(genres_namespace); Query queryAuthors(authors_namespace); Query queryBooks{Query(books_namespace, 0, 10).Where(price, CondGe, 500)}; queryBooks.OrInnerJoin(genreId_fk, genreid, CondEq, std::move(queryGenres)); queryBooks.LeftJoin(authorid_fk, authorid, CondEq, std::move(queryAuthors)); - - std::string dsl = queryBooks.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(queryBooks == testLoadDslQuery); + checkQueryDslParse(queryBooks); } TEST_F(JoinSelectsApi, EqualPositionDSLTest) { @@ -22,12 +25,7 @@ TEST_F(JoinSelectsApi, EqualPositionDSLTest) { query.OpenBracket().Where("f4", CondEq, 4).Where("f5", CondLt, 10); query.AddEqualPosition({"f4", "f5"}); query.CloseBracket(); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, MergedQueriesDSLTest) { @@ -37,28 +35,15 @@ TEST_F(JoinSelectsApi, MergedQueriesDSLTest) { mainBooksQuery.mergeQueries_.emplace_back(Merge, std::move(firstMergedQuery)); mainBooksQuery.mergeQueries_.emplace_back(Merge, std::move(secondMergedQuery)); - - std::string dsl = mainBooksQuery.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(mainBooksQuery == testLoadDslQuery); + checkQueryDslParse(mainBooksQuery); } TEST_F(JoinSelectsApi, AggregateFunctonsDSLTest) { Query query{Query(books_namespace, 10, 100).Where(pages, CondGe, 150)}; - query.aggregations_.push_back({AggAvg, {price}}); - query.aggregations_.push_back({AggSum, {pages}}); - query.aggregations_.push_back({AggFacet, {title, pages}, {{{title, true}}}, 100, 10}); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, SelectFilterDSLTest) { @@ -66,12 +51,7 @@ TEST_F(JoinSelectsApi, SelectFilterDSLTest) { query.selectFilter_.push_back(price); query.selectFilter_.push_back(pages); query.selectFilter_.push_back(title); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, SelectFilterInJoinDSLTest) { @@ -85,35 +65,18 @@ TEST_F(JoinSelectsApi, SelectFilterInJoinDSLTest) { queryBooks.LeftJoin(authorid_fk, authorid, CondEq, std::move(queryAuthors)); } - std::string dsl = queryBooks.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(queryBooks, testLoadDslQuery); + checkQueryDslParse(queryBooks); } TEST_F(JoinSelectsApi, ReqTotalDSLTest) { Query query{Query(books_namespace, 10, 100, ModeNoTotal).Where(pages, CondGe, 150)}; - - std::string dsl1 = query.GetJSON(); - Query testLoadDslQuery1; - Error err = testLoadDslQuery1.FromJSON(dsl1); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery1); + checkQueryDslParse(query); query.CachedTotal(); - std::string dsl2 = query.GetJSON(); - Query testLoadDslQuery2; - err = testLoadDslQuery2.FromJSON(dsl2); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery2); + checkQueryDslParse(query); query.ReqTotal(); - std::string dsl3 = query.GetJSON(); - Query testLoadDslQuery3; - err = testLoadDslQuery3.FromJSON(dsl3); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(query, testLoadDslQuery3); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, SelectFunctionsDSLTest) { @@ -121,22 +84,13 @@ TEST_F(JoinSelectsApi, SelectFunctionsDSLTest) { query.AddFunction("f1()"); query.AddFunction("f2()"); query.AddFunction("f3()"); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, CompositeValuesDSLTest) { std::string pagesBookidIndex = pages + std::string("+") + bookid; Query query{Query(books_namespace).WhereComposite(pagesBookidIndex.c_str(), CondGe, {{Variant(500), Variant(10)}})}; - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, GeneralDSLTest) { @@ -153,12 +107,7 @@ TEST_F(JoinSelectsApi, GeneralDSLTest) { testDslQuery.selectFilter_.push_back(authorid_fk); testDslQuery.AddFunction("f1()"); testDslQuery.AddFunction("f2()"); - testDslQuery.aggregations_.push_back({AggDistinct, {bookid}}); - Query testLoadDslQuery; - const std::string dsl1 = testDslQuery.GetJSON(); - Error err = testLoadDslQuery.FromJSON(dsl1); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(testDslQuery == testLoadDslQuery); + checkQueryDslParse(testDslQuery); } diff --git a/cpp_src/gtests/tests/unit/ft/ft_generic.cc b/cpp_src/gtests/tests/unit/ft/ft_generic.cc index ac58f97de..08ef6a7dc 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_generic.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_generic.cc @@ -113,6 +113,54 @@ TEST_P(FTGenericApi, CompositeSelectWithFields) { } } +TEST_P(FTGenericApi, MergeWithSameNSAndSelectFunctions) { + Init(GetDefaultConfig()); + AddInBothFields("An entity is something|"sv, "| that in exists entity as itself"sv); + AddInBothFields("In law, a legal entity is|"sv, "|an entity that is capable of something bearing legal rights"sv); + AddInBothFields("In politics, entity is used as|"sv, "| term for entity territorial divisions of some countries"sv); + + for (const auto& query : CreateAllPermutatedQueries("", {"*entity", "somethin*"}, "")) { + for (const auto& field : {std::string("ft1"), std::string("ft2")}) { + auto dsl = std::string("@").append(field).append(" ").append(query); + auto qr{reindexer::Query("nm1").Where("ft3", CondEq, dsl)}; + reindexer::QueryResults res; + auto mqr{reindexer::Query("nm1").Where("ft3", CondEq, std::move(dsl))}; + mqr.AddFunction(field + " = snippet(,\"\",3,2,,d)"); + + qr.mergeQueries_.emplace_back(Merge, std::move(mqr)); + qr.AddFunction(field + " = highlight(,)"); + auto err = rt.reindexer->Select(qr, res); + EXPECT_TRUE(err.ok()) << err.what(); + + std::unordered_set data{"An entity is something|"sv, + "An entity is something|d"sv, + "| that in exists entity as itself"sv, + "In law, a legal entity is|"sv, + "|an entity that is capable of something bearing legal rights"sv, + "an entity tdof something bd"sv, + "al entity id"sv, + "In politics, entity is used as|"sv, + "| term for entity territorial divisions of some countries"sv, + "ts entity ad"sv, + "s, entity id"sv, + "or entity td"sv}; + + rt.PrintQueryResults("nm1", res); + for (auto it : res) { + auto ritem(it.GetItem(false)); + for (auto idx = 1; idx < ritem.NumFields(); idx++) { + auto curField = ritem[idx].Name(); + if (curField != field) continue; + auto it = data.find(ritem[curField].As()); + ASSERT_TRUE(it != data.end()); + data.erase(it); + } + } + EXPECT_TRUE(data.empty()); + } + } +} + TEST_P(FTGenericApi, SelectWithPlus) { Init(GetDefaultConfig()); @@ -534,7 +582,8 @@ TEST_P(FTGenericApi, DeleteTest) { // Delete(data[1].first); // Delete(data[1].first); - Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + const auto err = Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + ASSERT_TRUE(err.ok()) << err.what(); res = SimpleSelect("entity"); // for (auto it : res) { @@ -572,7 +621,8 @@ TEST_P(FTGenericApi, RebuildAfterDeletion) { auto res = selectF("entity"); ASSERT_EQ(res.Count(), 3); - Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + err = Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + ASSERT_TRUE(err.ok()) << err.what(); res = selectF("entity"); ASSERT_EQ(res.Count(), 2); } diff --git a/cpp_src/gtests/tests/unit/ft/ft_stress.cc b/cpp_src/gtests/tests/unit/ft/ft_stress.cc index f25e5429c..e0a0d35e7 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_stress.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_stress.cc @@ -10,7 +10,9 @@ class FTStressApi : public FTApi { }; TEST_P(FTStressApi, BasicStress) { - Init(GetDefaultConfig()); + const std::string kStorage = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex_FTApi/BasicStress"); + reindexer::fs::RmDirAll(kStorage); + Init(GetDefaultConfig(), NS1, kStorage); std::vector data; std::vector phrase; @@ -29,7 +31,8 @@ TEST_P(FTStressApi, BasicStress) { std::thread statsThread([&] { while (!terminate) { reindexer::QueryResults qr; - rt.reindexer->Select(reindexer::Query("#memstats"), qr); + const auto err = rt.reindexer->Select(reindexer::Query("#memstats"), qr); + ASSERT_TRUE(err.ok()) << err.what(); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } }); @@ -89,7 +92,8 @@ TEST_P(FTStressApi, ConcurrencyCheck) { lck.unlock(); while (!terminate) { reindexer::QueryResults qr; - rt.reindexer->Select(reindexer::Query("#memstats"), qr); + const auto err = rt.reindexer->Select(reindexer::Query("#memstats"), qr); + ASSERT_TRUE(err.ok()) << err.what(); } }); } else { diff --git a/cpp_src/gtests/tests/unit/join_test.cc b/cpp_src/gtests/tests/unit/join_test.cc index 35f4f46f1..17aee3df6 100644 --- a/cpp_src/gtests/tests/unit/join_test.cc +++ b/cpp_src/gtests/tests/unit/join_test.cc @@ -4,6 +4,7 @@ #include #include "core/itemimpl.h" #include "core/nsselecter/joinedselector.h" +#include "core/type_consts_helpers.h" #include "join_on_conditions_api.h" #include "join_selects_api.h" #include "test_helpers.h" @@ -33,8 +34,8 @@ TEST_F(JoinSelectsApi, JoinsAsWhereConditionsTest) { QueryWatcher watcher{queryBooks}; reindexer::QueryResults qr; Error err = rt.reindexer->Select(queryBooks, qr); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr.Count() <= 50); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_LE(qr.Count(), 50); CheckJoinsInComplexWhereCondition(qr); } @@ -47,7 +48,7 @@ TEST_F(JoinSelectsApi, JoinsLockWithCache_364) { for (int i = 0; i < 10; ++i) { reindexer::QueryResults qr; Error err = rt.reindexer->Select(queryBooks, qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } } @@ -67,8 +68,8 @@ TEST_F(JoinSelectsApi, JoinsAsWhereConditionsTest2) { QueryWatcher watcher{query}; reindexer::QueryResults qr; Error err = rt.reindexer->Select(query, qr); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr.Count() <= 50); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_LE(qr.Count(), 50); CheckJoinsInComplexWhereCondition(qr); } @@ -91,14 +92,16 @@ TEST_F(JoinSelectsApi, SqlPasringTest) { Query dstQuery; dstQuery.FromSQL(wrser.Slice()); - ASSERT_TRUE(srcQuery == dstQuery); + ASSERT_EQ(srcQuery, dstQuery); wrser.Reset(); srcQuery.Serialize(wrser); Query deserializedQuery; reindexer::Serializer ser(wrser.Buf(), wrser.Len()); deserializedQuery.Deserialize(ser); - ASSERT_TRUE(srcQuery == deserializedQuery); + ASSERT_EQ(srcQuery, deserializedQuery) << "Original query:\n" + << srcQuery.GetSQL() << "\nDeserialized query:\n" + << deserializedQuery.GetSQL(); } TEST_F(JoinSelectsApi, InnerJoinTest) { @@ -109,14 +112,14 @@ TEST_F(JoinSelectsApi, InnerJoinTest) { reindexer::QueryResults joinQueryRes; Error err = rt.reindexer->Select(joinQuery, joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = VerifyResJSON(joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); reindexer::QueryResults pureSelectRes; err = rt.reindexer->Select(queryBooks, pureSelectRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); QueryResultRows joinSelectRows; QueryResultRows pureSelectRows; @@ -129,7 +132,7 @@ TEST_F(JoinSelectsApi, InnerJoinTest) { reindexer::QueryResults authorsSelectRes; Query authorsQuery{Query(authors_namespace).Where(authorid, CondEq, authorIdKeyRef)}; err = rt.reindexer->Select(authorsQuery, authorsSelectRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); if (err.ok()) { int bookId = booksItem[bookid].Get(); @@ -152,7 +155,7 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { Query booksQuery{Query(books_namespace).Where(price, CondGe, 500)}; reindexer::QueryResults booksQueryRes; Error err = rt.reindexer->Select(booksQuery, booksQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); QueryResultRows pureSelectRows; if (err.ok()) { @@ -169,10 +172,10 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { QueryWatcher watcher{joinQuery}; reindexer::QueryResults joinQueryRes; err = rt.reindexer->Select(joinQuery, joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = VerifyResJSON(joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); if (err.ok()) { std::unordered_set presentedAuthorIds; @@ -188,7 +191,7 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { for (auto joinedFieldIt = itemIt.begin(); joinedFieldIt != itemIt.end(); ++joinedFieldIt) { reindexer::ItemImpl item2(joinedFieldIt.GetItem(0, joinQueryRes.getPayloadType(1), joinQueryRes.getTagsMatcher(1))); Variant authorIdKeyRef2 = item2.GetField(joinQueryRes.getPayloadType(1).FieldByName(authorid_fk)); - EXPECT_TRUE(authorIdKeyRef1 == authorIdKeyRef2); + EXPECT_EQ(authorIdKeyRef1, authorIdKeyRef2); } presentedAuthorIds.insert(static_cast(authorIdKeyRef1)); @@ -208,15 +211,15 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { int authorId = static_cast(authorIdKeyRef1); auto itAutorid(presentedAuthorIds.find(authorId)); - EXPECT_TRUE(itAutorid != presentedAuthorIds.end()); + EXPECT_NE(itAutorid, presentedAuthorIds.end()); auto itRowidIndex(rowidsIndexes.find(rowid)); - EXPECT_TRUE(itRowidIndex != rowidsIndexes.end()); + EXPECT_NE(itRowidIndex, rowidsIndexes.end()); if (itRowidIndex != rowidsIndexes.end()) { Item item2((joinQueryRes.begin() + rowid).GetItem(false)); Variant authorIdKeyRef2 = item2[authorid]; - EXPECT_TRUE(authorIdKeyRef1 == authorIdKeyRef2); + EXPECT_EQ(authorIdKeyRef1, authorIdKeyRef2); } } } @@ -236,10 +239,10 @@ TEST_F(JoinSelectsApi, OrInnerJoinTest) { reindexer::QueryResults queryRes; Error err = rt.reindexer->Select(orInnerJoinQuery, queryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = VerifyResJSON(queryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); if (err.ok()) { for (auto rowIt : queryRes) { @@ -251,7 +254,7 @@ TEST_F(JoinSelectsApi, OrInnerJoinTest) { for (int i = 0; i < authorIdIt.ItemsCount(); ++i) { reindexer::ItemImpl authorsItem(authorIdIt.GetItem(i, queryRes.getPayloadType(1), queryRes.getTagsMatcher(1))); Variant authorIdKeyRef2 = authorsItem.GetField(queryRes.getPayloadType(1).FieldByName(authorid)); - EXPECT_TRUE(authorIdKeyRef1 == authorIdKeyRef2); + EXPECT_EQ(authorIdKeyRef1, authorIdKeyRef2); } reindexer::joins::JoinedFieldIterator genreIdIt = itemIt.at(genresNsJoinIndex); @@ -259,7 +262,7 @@ TEST_F(JoinSelectsApi, OrInnerJoinTest) { for (int i = 0; i < genreIdIt.ItemsCount(); ++i) { reindexer::ItemImpl genresItem = genreIdIt.GetItem(i, queryRes.getPayloadType(2), queryRes.getTagsMatcher(2)); Variant genresIdKeyRef2 = genresItem.GetField(queryRes.getPayloadType(2).FieldByName(genreid)); - EXPECT_TRUE(genresIdKeyRef1 == genresIdKeyRef2); + EXPECT_EQ(genresIdKeyRef1, genresIdKeyRef2); } } } @@ -292,7 +295,7 @@ TEST_F(JoinSelectsApi, JoinTestSorting) { for (auto rowIt : joinQueryRes) { Item item = rowIt.GetItem(false); if (!prevField.Type().Is()) { - ASSERT_TRUE(prevField.Compare(item[age]) <= 0); + ASSERT_LE(prevField.Compare(item[age]), 0); } Variant key = item[authorid]; @@ -304,14 +307,14 @@ TEST_F(JoinSelectsApi, JoinTestSorting) { for (int i = 0; i < joinedFieldIt.ItemsCount(); ++i) { reindexer::ItemImpl joinItem(joinedFieldIt.GetItem(i, joinQueryRes.getPayloadType(1), joinQueryRes.getTagsMatcher(1))); Variant fkey = joinItem.GetField(joinQueryRes.getPayloadType(1).FieldByName(authorid_fk)); - ASSERT_TRUE(key.Compare(fkey) == 0) << key.As() << " " << fkey.As(); + ASSERT_EQ(key.Compare(fkey), 0) << key.As() << " " << fkey.As(); Variant recentJoinedValue = joinItem.GetField(joinQueryRes.getPayloadType(1).FieldByName(price)); - ASSERT_TRUE(recentJoinedValue.As() >= 200); + ASSERT_GE(recentJoinedValue.As(), 200); if (!prevJoinedValue.Type().Is()) { - ASSERT_TRUE(prevJoinedValue.Compare(recentJoinedValue) >= 0); + ASSERT_GE(prevJoinedValue.Compare(recentJoinedValue), 0); } Variant pagesValue = joinItem.GetField(joinQueryRes.getPayloadType(1).FieldByName(pages)); - ASSERT_TRUE(pagesValue.As() >= 100); + ASSERT_GE(pagesValue.As(), 100); prevJoinedValue = recentJoinedValue; } prevField = item[age]; @@ -350,7 +353,7 @@ TEST_F(JoinSelectsApi, TestSortingByJoinedNs) { const Variant recentValue = joinItem.GetField(joinQueryRes2.getPayloadType(1).FieldByName(age)); if (!prevValue.Type().Is()) { reindexer::WrSerializer ser; - ASSERT_TRUE(prevValue.Compare(recentValue) <= 0) << (prevValue.Dump(ser), ser << ' ', recentValue.Dump(ser), ser.Slice()); + ASSERT_LE(prevValue.Compare(recentValue), 0) << (prevValue.Dump(ser), ser << ' ', recentValue.Dump(ser), ser.Slice()); } prevValue = recentValue; } @@ -365,12 +368,12 @@ TEST_F(JoinSelectsApi, JoinTestSelectNonIndexedField) { qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << err.what(); + ASSERT_EQ(qr.Count(), 1); Item theOnlyItem = qr[0].GetItem(false); VariantArray krefs = theOnlyItem[title]; - ASSERT_TRUE(krefs.size() == 1); - ASSERT_TRUE(krefs[0].As() == "Crime and Punishment"); + ASSERT_EQ(krefs.size(), 1); + ASSERT_EQ(krefs[0].As(), "Crime and Punishment"); } TEST_F(JoinSelectsApi, JoinByNonIndexedField) { @@ -400,7 +403,7 @@ TEST_F(JoinSelectsApi, JoinByNonIndexedField) { qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << err.what(); + ASSERT_EQ(qr.Count(), 1); // And backwards even! reindexer::QueryResults qr2; @@ -411,7 +414,7 @@ TEST_F(JoinSelectsApi, JoinByNonIndexedField) { qr2); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr2.Count() == 1) << err.what(); + ASSERT_EQ(qr2.Count(), 1); } TEST_F(JoinSelectsApi, JoinsEasyStressTest) { @@ -426,15 +429,15 @@ TEST_F(JoinSelectsApi, JoinsEasyStressTest) { for (size_t i = 0; i < 10; ++i) { reindexer::QueryResults queryRes; Error err = rt.reindexer->Select(orInnerJoinQuery, queryRes); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(queryRes.Count() > 0); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_GT(queryRes.Count(), 0); } }; auto removeTh = [this]() { QueryResults qres; Error err = rt.reindexer->Delete(Query(books_namespace, 0, 10).Where(price, CondGe, 5000), qres); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); }; int32_t since = 0, count = 1000; @@ -493,22 +496,20 @@ TEST_F(JoinSelectsApi, JoinPreResultStoreValuesOptimizationStressTest) { QueryResults qres; while (!start) std::this_thread::sleep_for(std::chrono::milliseconds(1)); Error err = rt.reindexer->Select(q, qres); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); }); } start = true; for (auto& th : threads) th.join(); } -static bool checkForAllowedJsonTags(const std::vector& tags, gason::JsonValue jsonValue) { +static void checkForAllowedJsonTags(const std::vector& tags, gason::JsonValue jsonValue) { size_t count = 0; for (const auto& elem : jsonValue) { - if (std::find(tags.begin(), tags.end(), std::string_view(elem.key)) == tags.end()) { - return false; - } + ASSERT_NE(std::find(tags.begin(), tags.end(), std::string_view(elem.key)), tags.end()); ++count; } - return (count == tags.size()); + ASSERT_EQ(count, tags.size()); } TEST_F(JoinSelectsApi, JoinWithSelectFilter) { @@ -525,22 +526,26 @@ TEST_F(JoinSelectsApi, JoinWithSelectFilter) { ASSERT_TRUE(err.ok()) << err.what(); for (auto it : qr) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer wrser; - it.GetJSON(wrser, false); + err = it.GetJSON(wrser, false); + ASSERT_TRUE(err.ok()) << err.what(); reindexer::joins::ItemIterator joinIt = it.GetJoined(); gason::JsonParser jsonParser; gason::JsonNode root = jsonParser.Parse(reindexer::giftStr(wrser.Slice())); - EXPECT_TRUE(checkForAllowedJsonTags({title, price, "joined_authors_namespace"}, root.value)); + checkForAllowedJsonTags({title, price, "joined_authors_namespace"}, root.value); for (auto fieldIt = joinIt.begin(); fieldIt != joinIt.end(); ++fieldIt) { QueryResults jqr = fieldIt.ToQueryResults(); jqr.addNSContext(qr.getPayloadType(1), qr.getTagsMatcher(1), qr.getFieldsFilter(1), qr.getSchema(1)); for (auto jit : jqr) { + ASSERT_TRUE(jit.Status().ok()) << jit.Status().what(); wrser.Reset(); - jit.GetJSON(wrser, false); + err = jit.GetJSON(wrser, false); + ASSERT_TRUE(err.ok()) << err.what(); root = jsonParser.Parse(reindexer::giftStr(wrser.Slice())); - EXPECT_TRUE(checkForAllowedJsonTags({name, age}, root.value)); + checkForAllowedJsonTags({name, age}, root.value); } } } @@ -573,7 +578,7 @@ TEST_F(JoinSelectsApi, TestMergeWithJoins) { for (auto it : qr) { Item item = it.GetItem(false); auto joined = it.GetJoined(); - ASSERT_TRUE(joined.getJoinedFieldsCount() == 1); + ASSERT_EQ(joined.getJoinedFieldsCount(), 1); bool booksItem = (rowId <= 10000); QueryResults jqr = joined.begin().ToQueryResults(); @@ -585,14 +590,14 @@ TEST_F(JoinSelectsApi, TestMergeWithJoins) { for (auto jit : jqr) { Item jItem = jit.GetItem(false); Variant value = jItem[authorid]; - ASSERT_TRUE(value == fkValue); + ASSERT_EQ(value, fkValue); } } else { Variant fkValue = item[locationid_fk]; for (auto jit : jqr) { Item jItem = jit.GetItem(false); Variant value = jItem[locationid]; - ASSERT_TRUE(value == fkValue); + ASSERT_EQ(value, fkValue); } } @@ -634,30 +639,29 @@ TEST_F(JoinSelectsApi, TestNestedMergesInMergesError) { TEST_F(JoinOnConditionsApi, TestGeneralConditions) { const std::string sqlTemplate = R"(select * from books_namespace inner join books_namespace on (books_namespace.authorid_fk = books_namespace.authorid_fk and books_namespace.pages %s books_namespace.pages);)"; - std::vector conditionsSet = {CondLt, CondLe, CondGt, CondGe, CondEq}; - for (size_t i = 0; i < conditionsSet.size(); ++i) { - CondType condition = conditionsSet[i]; + for (CondType condition : {CondLt, CondLe, CondGt, CondGe, CondEq}) { Query queryBooks; queryBooks.FromSQL(GetSql(sqlTemplate, condition)); QueryResults qr; Error err = rt.reindexer->Select(queryBooks, qr); ASSERT_TRUE(err.ok()) << err.what(); for (auto it : qr) { - auto item = it.GetItem(); + const auto item = it.GetItem(); ASSERT_TRUE(item.Status().ok()) << item.Status().what(); - auto joined = it.GetJoined(); - ASSERT_TRUE(joined.getJoinedFieldsCount() == 1); + const Variant authorid1 = item[authorid_fk]; + const Variant pages1 = item[pages]; + const auto joined = it.GetJoined(); + ASSERT_EQ(joined.getJoinedFieldsCount(), 1); QueryResults jqr = joined.begin().ToQueryResults(); jqr.addNSContext(qr.getPayloadType(0), qr.getTagsMatcher(0), qr.getFieldsFilter(0), qr.getSchema(0)); for (auto jit : jqr) { auto joinedItem = jit.GetItem(); ASSERT_TRUE(joinedItem.Status().ok()) << joinedItem.Status().what(); - Variant authorid1 = item[authorid_fk]; Variant authorid2 = joinedItem[authorid_fk]; - ASSERT_TRUE(authorid1 == authorid2); - Variant pages1 = item[pages]; + ASSERT_EQ(authorid1, authorid2); Variant pages2 = joinedItem[pages]; - ASSERT_TRUE(CompareVariants(pages1, pages2, condition)); + ASSERT_TRUE(CompareVariants(pages1, pages2, condition)) + << pages1.As() << ' ' << reindexer::CondTypeToStr(condition) << ' ' << pages2.As(); } } } @@ -691,14 +695,14 @@ TEST_F(JoinOnConditionsApi, TestComparisonConditions) { auto item1 = it1.GetItem(); ASSERT_TRUE(item1.Status().ok()) << item1.Status().what(); auto joined1 = it1.GetJoined(); - ASSERT_TRUE(joined1.getJoinedFieldsCount() == 1); + ASSERT_EQ(joined1.getJoinedFieldsCount(), 1); QueryResults jqr1 = joined1.begin().ToQueryResults(); jqr1.addNSContext(qr1.getPayloadType(1), qr1.getTagsMatcher(1), qr1.getFieldsFilter(1), qr1.getSchema(0)); auto item2 = it2.GetItem(); ASSERT_TRUE(item2.Status().ok()) << item2.Status().what(); auto joined2 = it2.GetJoined(); - ASSERT_TRUE(joined2.getJoinedFieldsCount() == 1); + ASSERT_EQ(joined2.getJoinedFieldsCount(), 1); QueryResults jqr2 = joined2.begin().ToQueryResults(); jqr2.addNSContext(qr2.getPayloadType(1), qr2.getTagsMatcher(1), qr2.getFieldsFilter(1), qr2.getSchema(0)); @@ -748,8 +752,10 @@ TEST_F(JoinOnConditionsApi, TestLeftJoinOnCondSet) { ASSERT_EQ(qr.Count(), results.size()); int k = 0; for (auto it = qr.begin(); it != qr.end(); ++it, ++k) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer ser; - it.GetJSON(ser, false); + err = it.GetJSON(ser, false); + ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(ser.c_str(), results[k]); } }; @@ -787,7 +793,7 @@ TEST_F(JoinOnConditionsApi, TestInvalidConditions) { } QueryResults qr; Error err = rt.reindexer->Select(Query(books_namespace).InnerJoin(authorid_fk, authorid, CondAllSet, Query(authors_namespace)), qr); - EXPECT_TRUE(!err.ok()); + EXPECT_FALSE(err.ok()); err = rt.reindexer->Select(Query(books_namespace).InnerJoin(authorid_fk, authorid, CondLike, Query(authors_namespace)), qr); - EXPECT_TRUE(!err.ok()); + EXPECT_FALSE(err.ok()); } diff --git a/cpp_src/gtests/tests/unit/namespace_test.cc b/cpp_src/gtests/tests/unit/namespace_test.cc index 4de15ddae..e8bffaa96 100644 --- a/cpp_src/gtests/tests/unit/namespace_test.cc +++ b/cpp_src/gtests/tests/unit/namespace_test.cc @@ -339,12 +339,12 @@ TEST_F(NsApi, QueryperfstatsNsDummyTest) { QueryResults qr; err = rt.reindexer->Select(Query("#queriesperfstats"), qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0) << "#queriesperfstats table is empty!"; + ASSERT_GT(qr.Count(), 0) << "#queriesperfstats table is empty!"; for (size_t i = 0; i < qr.Count(); ++i) { std::cout << qr[i].GetItem(false).GetJSON() << std::endl; } } - ASSERT_TRUE(qres.Count() == 1) << "Expected 1 row for this query, got " << qres.Count(); + ASSERT_EQ(qres.Count(), 1); Item item = qres[0].GetItem(false); Variant val; val = item["latency_stddev"]; @@ -415,12 +415,12 @@ TEST_F(NsApi, TestUpdateNonindexedField) { Query updateQuery{Query(default_namespace).Where("id", CondGe, Variant("1500")).Set("nested.bonus", static_cast(100500))}; Error err = rt.reindexer->Update(updateQuery, qrUpdate); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() == 500) << qrUpdate.Count(); + ASSERT_EQ(qrUpdate.Count(), 500); QueryResults qrAll; err = rt.reindexer->Select(Query(default_namespace).Where("id", CondGe, Variant("1500")), qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == 500) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), 500); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -439,12 +439,12 @@ TEST_F(NsApi, TestUpdateSparseField) { Query updateQuery{Query(default_namespace).Where("id", CondGe, Variant("1500")).Set("sparse_field", static_cast(100500))}; Error err = rt.reindexer->Update(updateQuery, qrUpdate); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() == 500) << qrUpdate.Count(); + ASSERT_EQ(qrUpdate.Count(), 500); QueryResults qrAll; err = rt.reindexer->Select(Query(default_namespace).Where("id", CondGe, Variant("1500")), qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == 500) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), 500); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -473,7 +473,7 @@ TEST_F(NsApi, TestUpdateTwoFields) { // Make sure query worked well ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() == 1) << qrUpdate.Count(); + ASSERT_EQ(qrUpdate.Count(), 1); // Make sure: // 1. JSON of the item is correct @@ -500,12 +500,12 @@ static void updateArrayField(const std::shared_ptr &reinde Query updateQuery{Query(ns).Where("id", CondGe, Variant("500")).Set(updateFieldPath, values)}; Error err = reindexer->Update(updateQuery, qrUpdate); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() > 0) << qrUpdate.Count(); + ASSERT_GT(qrUpdate.Count(), 0); QueryResults qrAll; err = reindexer->Select(Query(ns).Where("id", CondGe, Variant("500")), qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == qrUpdate.Count()) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), qrUpdate.Count()); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -537,7 +537,7 @@ TEST_F(NsApi, TestUpdateNonindexedArrayField2) { QueryResults qr; Error err = rt.reindexer->Select(R"(update test_namespace set nested.bonus=[{"first":1,"second":2,"third":3}] where id = 1000;)", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); std::string_view json = item.GetJSON(); @@ -553,7 +553,7 @@ TEST_F(NsApi, TestUpdateNonindexedArrayField3) { Error err = rt.reindexer->Select(R"(update test_namespace set nested.bonus=[{"id":1},{"id":2},{"id":3},{"id":4}] where id = 1000;)", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); VariantArray val = item["nested.bonus"]; @@ -576,12 +576,12 @@ TEST_F(NsApi, TestUpdateNonindexedArrayField4) { QueryResults qr; Error err = rt.reindexer->Select(R"(update test_namespace set nested.bonus=[0] where id = 1000;)", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); std::string_view json = item.GetJSON(); size_t pos = json.find(R"("nested":{"bonus":[0])"); - ASSERT_TRUE(pos != std::string::npos) << "'nested.bonus' was not updated properly" << json; + ASSERT_NE(pos, std::string::npos) << "'nested.bonus' was not updated properly" << json; } TEST_F(NsApi, TestUpdateNonindexedArrayField5) { @@ -611,12 +611,12 @@ TEST_F(NsApi, TestUpdateIndexedArrayField2) { Query q{Query(default_namespace).Where(idIdxName, CondEq, static_cast(1000)).Set(indexedArrayField, std::move(value.MarkArray()))}; Error err = rt.reindexer->Update(q, qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); std::string_view json = item.GetJSON(); size_t pos = json.find(R"("indexed_array_field":[77])"); - ASSERT_TRUE(pos != std::string::npos) << "'indexed_array_field' was not updated properly" << json; + ASSERT_NE(pos, std::string::npos) << "'indexed_array_field' was not updated properly" << json; } static void addAndSetNonindexedField(const std::shared_ptr &reindexer, const std::string &ns, @@ -844,6 +844,7 @@ TEST_F(NsApi, DropArrayField3) { DropArrayItem(rt.reindexer, default_namespace, "nested.nested_array[*].prices[*]", "nested.nested_array.prices"); } +#if (0) // #1500 TEST_F(NsApi, DropArrayField4) { // 1. Define NS // 2. Fill NS @@ -853,6 +854,7 @@ TEST_F(NsApi, DropArrayField4) { DropArrayItem(rt.reindexer, default_namespace, "nested.nested_array[0].prices[((2+4)*2)/6]", "nested.nested_array.prices", 0, ((2 + 4) * 2) / 6); } +#endif TEST_F(NsApi, SetArrayFieldWithSql) { // 1. Define NS @@ -1692,12 +1694,12 @@ static void checkFieldConversion(const std::shared_ptr &re ASSERT_TRUE(!err.ok()); } else { ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() > 0) << qrUpdate.Count(); + ASSERT_GT(qrUpdate.Count(), 0); QueryResults qrAll; err = reindexer->Select(selectQuery, qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == qrUpdate.Count()) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), qrUpdate.Count()); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -1845,7 +1847,7 @@ TEST_F(NsApi, TestUpdatePkFieldNoConditions) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set id = id + 1;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0); + ASSERT_GT(qr.Count(), 0); int i = 1; for (auto &it : qr) { @@ -1862,7 +1864,7 @@ TEST_F(NsApi, TestUpdateIndexArrayWithNull) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set indexed_array_field = null where id = 1;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); for (auto &it : qr) { Item item = it.GetItem(false); @@ -1983,7 +1985,7 @@ TEST_F(NsApi, TestUpdateNonIndexFieldWithNull) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set extra = null where id = 1001;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); for (auto &it : qr) { Item item = it.GetItem(false); @@ -2008,7 +2010,7 @@ TEST_F(NsApi, TestUpdateEmptyArrayField) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set indexed_array_field = [] where id = 1;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); Variant idFieldVal = item[idIdxName]; @@ -2070,12 +2072,12 @@ TEST_F(NsApi, TestUpdateEmptyIndexedField) { .Set(indexedArrayField, {Variant(static_cast(4)), Variant(static_cast(5)), Variant(static_cast(6))}); Error err = rt.reindexer->Update(q, qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); QueryResults qr2; err = rt.reindexer->Select("select * from test_namespace where id = 1001;", qr2); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr2.Count() == 1); + ASSERT_EQ(qr2.Count(), 1); for (auto it : qr2) { Item item = it.GetItem(false); @@ -2100,7 +2102,7 @@ TEST_F(NsApi, TestDropField) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace drop extra where id >= 1000 and id < 1010;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 10) << qr.Count(); + ASSERT_EQ(qr.Count(), 10); for (auto it : qr) { Item item = it.GetItem(false); @@ -2112,7 +2114,7 @@ TEST_F(NsApi, TestDropField) { QueryResults qr2; err = rt.reindexer->Select("update test_namespace drop nested.bonus where id >= 1005 and id < 1010;", qr2); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr2.Count() == 5); + ASSERT_EQ(qr2.Count(), 5); for (auto it : qr2) { Item item = it.GetItem(false); @@ -2144,7 +2146,7 @@ TEST_F(NsApi, TestUpdateFieldWithFunction) { Error err = rt.reindexer->Select( "update test_namespace set int_field = SERIAL(), extra = SERIAL(), nested.timeField = NOW(msec) where id >= 0;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0); + ASSERT_GT(qr.Count(), 0); int i = 1; for (auto &it : qr) { @@ -2168,7 +2170,7 @@ TEST_F(NsApi, TestUpdateFieldWithExpressions) { "0;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0) << qr.Count(); + ASSERT_GT(qr.Count(), 0); int i = 1; for (auto &it : qr) { @@ -2207,22 +2209,22 @@ static void checkQueryDsl(const Query &src) { } } if (objectValues) { - EXPECT_TRUE(src.entries == dst.entries); - EXPECT_TRUE(src.aggregations_ == dst.aggregations_); - EXPECT_TRUE(src._namespace == dst._namespace); - EXPECT_TRUE(src.sortingEntries_ == dst.sortingEntries_); - EXPECT_TRUE(src.calcTotal == dst.calcTotal); - EXPECT_TRUE(src.start == dst.start); - EXPECT_TRUE(src.count == dst.count); - EXPECT_TRUE(src.debugLevel == dst.debugLevel); - EXPECT_TRUE(src.strictMode == dst.strictMode); - EXPECT_TRUE(src.forcedSortOrder_ == dst.forcedSortOrder_); - EXPECT_TRUE(src.selectFilter_ == dst.selectFilter_); - EXPECT_TRUE(src.selectFunctions_ == dst.selectFunctions_); - EXPECT_TRUE(src.joinQueries_ == dst.joinQueries_); - EXPECT_TRUE(src.mergeQueries_ == dst.mergeQueries_); + EXPECT_EQ(src.entries, dst.entries); + EXPECT_EQ(src.aggregations_, dst.aggregations_); + EXPECT_EQ(src.NsName(), dst.NsName()); + EXPECT_EQ(src.sortingEntries_, dst.sortingEntries_); + EXPECT_EQ(src.CalcTotal(), dst.CalcTotal()); + EXPECT_EQ(src.Offset(), dst.Offset()); + EXPECT_EQ(src.Limit(), dst.Limit()); + EXPECT_EQ(src.debugLevel, dst.debugLevel); + EXPECT_EQ(src.strictMode, dst.strictMode); + EXPECT_EQ(src.forcedSortOrder_, dst.forcedSortOrder_); + EXPECT_EQ(src.selectFilter_, dst.selectFilter_); + EXPECT_EQ(src.selectFunctions_, dst.selectFunctions_); + EXPECT_EQ(src.joinQueries_, dst.joinQueries_); + EXPECT_EQ(src.mergeQueries_, dst.mergeQueries_); } else { - EXPECT_TRUE(dst == src); + EXPECT_EQ(dst, src); } } @@ -2379,12 +2381,13 @@ TEST_F(NsApi, MsgPackEncodingTest) { ASSERT_TRUE(err.ok()) << err.what(); std::string json(item.GetJSON()); - ASSERT_TRUE(json == items[i++]); + ASSERT_EQ(json, items[i++]); } reindexer::WrSerializer wrSer3; for (size_t i = 0; i < qr.Count(); ++i) { - qr[i].GetMsgPack(wrSer3, false); + const auto err = qr[i].GetMsgPack(wrSer3, false); + ASSERT_TRUE(err.ok()) << err.what(); } i = 0; @@ -2397,7 +2400,7 @@ TEST_F(NsApi, MsgPackEncodingTest) { ASSERT_TRUE(err.ok()) << err.what(); std::string json(item.GetJSON()); - ASSERT_TRUE(json == items[i++]); + ASSERT_EQ(json, items[i++]); } } diff --git a/cpp_src/gtests/tests/unit/queries_test.cc b/cpp_src/gtests/tests/unit/queries_test.cc index 5a9ac6eb7..725fa47d6 100644 --- a/cpp_src/gtests/tests/unit/queries_test.cc +++ b/cpp_src/gtests/tests/unit/queries_test.cc @@ -32,7 +32,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { auto& items = insertedItems_[default_namespace]; for (auto it = items.begin(); it != items.end();) { Error err = rt.reindexer->Delete(default_namespace, it->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); it = items.erase(it); if (++itemsCount == 4000) break; } @@ -43,7 +43,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { itemsCount = 0; for (auto it = items.begin(); it != items.end();) { Error err = rt.reindexer->Delete(default_namespace, it->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); it = items.erase(it); if (++itemsCount == 5000) break; } @@ -52,7 +52,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { auto itToRemove = items.begin(); if (itToRemove != items.end()) { Error err = rt.reindexer->Delete(default_namespace, itToRemove->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); items.erase(itToRemove); } FillDefaultNamespace(rand() % 100, 1, 0); @@ -62,7 +62,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { std::advance(itToRemove, rand() % std::min(100, int(items.size()))); if (itToRemove != items.end()) { Error err = rt.reindexer->Delete(default_namespace, itToRemove->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); items.erase(itToRemove); } } @@ -70,7 +70,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { for (auto it = items.begin(); it != items.end();) { Error err = rt.reindexer->Delete(default_namespace, it->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); it = items.erase(it); } @@ -105,6 +105,18 @@ TEST_F(QueriesApi, QueriesConditions) { CheckConditions(); } +#if !defined(REINDEX_WITH_TSAN) +TEST_F(QueriesApi, UuidQueries) { + FillUUIDNs(); + // hack to obtain not index not string uuid fields + /*auto err = rt.reindexer->DropIndex(uuidNs, {kFieldNameUuidNotIndex2}); // TODO uncomment this #1470 + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->DropIndex(uuidNs, {kFieldNameUuidNotIndex3}); + ASSERT_TRUE(err.ok()) << err.what();*/ + CheckUUIDQueries(); +} +#endif + TEST_F(QueriesApi, IndexCacheInvalidationTest) { std::vector> data{{0, 10}, {1, 9}, {2, 8}, {3, 7}, {4, 6}, {5, 5}, {6, 4}, {7, 3}, {8, 2}, {9, 1}, {10, 0}, {11, -1}}; @@ -340,15 +352,15 @@ TEST_F(QueriesApi, StrictModeTest) { const std::string kNotExistingField = "some_random_name123"; QueryResults qr; { - Query query = Query(testSimpleNs).Where(kNotExistingField, CondEmpty, 0); + Query query = Query(testSimpleNs).Where(kNotExistingField, CondEmpty, {}); Error err = rt.reindexer->Select(query.Strict(StrictModeNames), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeIndexes), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeNone), qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); Verify(qr, Query(testSimpleNs), *rt.reindexer); qr.Clear(); } @@ -356,13 +368,13 @@ TEST_F(QueriesApi, StrictModeTest) { { Query query = Query(testSimpleNs).Where(kNotExistingField, CondEq, 0); Error err = rt.reindexer->Select(query.Strict(StrictModeNames), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeIndexes), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeNone), qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); EXPECT_EQ(qr.Count(), 0); } } @@ -453,6 +465,7 @@ TEST_F(QueriesApi, JoinByNotIndexField) { ASSERT_EQ(qr.Count(), sizeof(expectedIds) / sizeof(int)); for (size_t i = 0; i < qr.Count(); ++i) { Item item = qr[i].GetItem(false); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); VariantArray values = item["id"]; ASSERT_EQ(values.size(), 1); EXPECT_EQ(values[0].As(), expectedIds[i]); @@ -807,14 +820,18 @@ TEST_F(QueriesApi, ConvertationStringToDoubleDuringSorting) { std::string print(const reindexer::Query& q, reindexer::QueryResults::Iterator& currIt, reindexer::QueryResults::Iterator& prevIt, const reindexer::QueryResults& qr) { + assertrx(currIt.Status().ok()); std::string res = '\n' + q.GetSQL() + "\ncurr: "; reindexer::WrSerializer ser; - currIt.GetJSON(ser, false); + const auto err = currIt.GetJSON(ser, false); + assertrx(err.ok()); res += ser.Slice(); if (prevIt != qr.end()) { + assertrx(prevIt.Status().ok()); res += "\nprev: "; ser.Reset(); - prevIt.GetJSON(ser, false); + const auto err = prevIt.GetJSON(ser, false); + assertrx(err.ok()); res += ser.Slice(); } return res; diff --git a/cpp_src/gtests/tests/unit/replication_master_master_test.cc b/cpp_src/gtests/tests/unit/replication_master_master_test.cc index 96aa46cb4..b620b6109 100644 --- a/cpp_src/gtests/tests/unit/replication_master_master_test.cc +++ b/cpp_src/gtests/tests/unit/replication_master_master_test.cc @@ -431,6 +431,105 @@ TEST_F(ReplicationSlaveSlaveApi, TransactionTest) { for (auto& node : nodes) node.Stop(); } +TEST_F(ReplicationSlaveSlaveApi, TransactionCopyPolicyForceSync) { + // Check transactions copy policy after force sync + /* + m + | + 1 + | + 2 + */ + constexpr std::string_view kJsonCfgNss = R"=({ + "namespaces": [ + { + "namespace": "*", + "start_copy_policy_tx_size": 10000, + "copy_policy_multiplier": 5, + "tx_size_to_always_copy": 100000 + }, + { + "namespace": "ns1", + "start_copy_policy_tx_size": 10000, + "copy_policy_multiplier": 5, + "tx_size_to_always_copy": 1 + } + ], + "type": "namespaces" + })="; + constexpr int port = 9999; + const std::string kBaseDbPath(fs::JoinPath(kBaseTestsetDbPath, "TransactionCopyPolicyForceSync")); + const std::string kDbPathMaster(kBaseDbPath + "/test_"); + constexpr int serverId = 5; + constexpr size_t kRows = 100; + const std::string nsName("ns1"); + + std::vector slaveConfiguration = {-1, port, port + 1}; + std::vector nodes; + for (size_t i = 0; i < slaveConfiguration.size(); i++) { + nodes.emplace_back(); + nodes.back().InitServer(i, port + i, port + 1000 + i, kDbPathMaster + std::to_string(i), "db", true); + nodes.back().Get()->EnableAllProfilings(); + } + + // Set tx copy policy for the node '2' to 'always copy' + { + auto item = nodes[2].Get()->api.reindexer->NewItem("#config"); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + auto err = item.FromJSON(kJsonCfgNss); + ASSERT_TRUE(err.ok()) << err.what(); + err = nodes[2].Get()->api.reindexer->Upsert("#config", item); + ASSERT_TRUE(err.ok()) << err.what(); + } + + for (size_t i = 0; i < slaveConfiguration.size(); i++) { + if (i == 0) { + ReplicationConfigTest config("master"); + nodes[i].Get()->MakeMaster(config); + } else { + std::string masterDsn = "cproto://127.0.0.1:" + std::to_string(slaveConfiguration[i]) + "/db"; + ReplicationConfigTest config("slave", false, true, serverId + i, masterDsn); + nodes[i].Get()->MakeSlave(slaveConfiguration[i], config); + } + } + nodes[2].Drop(); + + ServerControl& master = nodes[0]; + TestNamespace1 ns1(master, nsName); + WaitSync(nodes[0], nodes[1], nsName); + + // Restart node '2' + nodes[2].InitServer(2, port + 2, port + 1000 + 2, kDbPathMaster + std::to_string(2), "db", true); + std::string masterDsn = "cproto://127.0.0.1:" + std::to_string(slaveConfiguration[2]) + "/db"; + ReplicationConfigTest config("slave", false, true, serverId + 2, masterDsn); + nodes[2].Get()->MakeSlave(1, config); + WaitSync(nodes[0], nodes[2], nsName); + + // Apply tx + reindexer::client::SyncCoroTransaction tr = master.Get()->api.reindexer->NewTransaction(nsName); + for (unsigned int i = 0; i < kRows; i++) { + reindexer::client::Item item = tr.NewItem(); + auto err = item.FromJSON("{\"id\":" + std::to_string(i + kRows * 10) + "}"); + tr.Upsert(std::move(item)); + } + master.Get()->api.reindexer->CommitTransaction(tr); + WaitSync(nodes[0], nodes[2], nsName); + + // Check copy tx event in the perfstats + client::SyncCoroQueryResults qr(nodes[2].Get()->api.reindexer.get()); + auto err = nodes[2].Get()->api.reindexer->Select("select * from #perfstats", qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.Count(), 1); + WrSerializer ser; + err = qr.begin().GetJSON(ser, false); + ASSERT_TRUE(err.ok()) << err.what(); + gason::JsonParser parser; + auto resJS = parser.Parse(ser.Slice()); + ASSERT_EQ(resJS["transactions"]["total_copy_count"].As(-1), 1) << ser.Slice(); + + for (auto& node : nodes) node.Stop(); +} + TEST_F(ReplicationSlaveSlaveApi, ForceSync3Node) { // Check force-sync for cascade setup /* diff --git a/cpp_src/gtests/tests/unit/rpcclient_test.cc b/cpp_src/gtests/tests/unit/rpcclient_test.cc index a6c1027ee..9ea0b80b9 100644 --- a/cpp_src/gtests/tests/unit/rpcclient_test.cc +++ b/cpp_src/gtests/tests/unit/rpcclient_test.cc @@ -20,11 +20,11 @@ TEST_F(RPCClientTestApi, ConnectTimeout) { config.RequestTimeout = seconds(5); reindexer::client::Reindexer rx(config); auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); EXPECT_EQ(res.code(), errTimeout); res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, RequestTimeout) { @@ -35,14 +35,14 @@ TEST_F(RPCClientTestApi, RequestTimeout) { config.RequestTimeout = seconds(3); reindexer::client::Reindexer rx(config); auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); const std::string kNamespaceName = "MyNamespace"; res = rx.AddNamespace(reindexer::NamespaceDef(kNamespaceName)); EXPECT_EQ(res.code(), errTimeout); res = rx.DropNamespace(kNamespaceName); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, RequestCancels) { @@ -50,7 +50,7 @@ TEST_F(RPCClientTestApi, RequestCancels) { StartServer(); reindexer::client::Reindexer rx; auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); { CancelRdxContext ctx; @@ -72,7 +72,7 @@ TEST_F(RPCClientTestApi, RequestCancels) { } res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, SuccessfullRequestWithTimeout) { @@ -83,22 +83,23 @@ TEST_F(RPCClientTestApi, SuccessfullRequestWithTimeout) { config.RequestTimeout = seconds(6); reindexer::client::Reindexer rx(config); auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, ErrorLoginResponse) { AddFakeServer(); StartServer(kDefaultRPCServerAddr, errForbidden); reindexer::client::Reindexer rx; - rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - auto res = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); - EXPECT_EQ(res.code(), errForbidden) << res.what(); - res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); + ASSERT_TRUE(err.ok()) << err.what(); + err = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); + EXPECT_EQ(err.code(), errForbidden) << err.what(); + err = StopServer(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, SeveralDsnReconnect) { @@ -126,7 +127,7 @@ TEST_F(RPCClientTestApi, SeveralDsnReconnect) { connectData.emplace_back(std::string().append(cprotoIdentifier).append(uri).append(dbName), reindexer::client::ConnectOpts()); } auto res = rx.Connect(connectData); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); for (size_t i = 0; i < 100; ++i) { if (CheckIfFakeServerConnected(uris[0])) break; @@ -147,7 +148,7 @@ TEST_F(RPCClientTestApi, SeveralDsnReconnect) { if (res.ok()) break; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } StopAllServers(); } @@ -209,7 +210,7 @@ TEST_F(RPCClientTestApi, SelectFromClosedNamespace) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, RenameNamespace) { @@ -288,7 +289,7 @@ TEST_F(RPCClientTestApi, RenameNamespace) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroRequestTimeout) { @@ -305,19 +306,19 @@ TEST_F(RPCClientTestApi, CoroRequestTimeout) { config.RequestTimeout = seconds(1); reindexer::client::CoroReindexer rx(config); auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db", loop); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); const std::string kNamespaceName = "MyNamespace"; err = rx.AddNamespace(reindexer::NamespaceDef(kNamespaceName)); EXPECT_EQ(err.code(), errTimeout); loop.sleep(std::chrono::seconds(4)); err = rx.DropNamespace(kNamespaceName); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); finished = true; }); loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } static std::chrono::seconds GetMaxTimeForCoroSelectTimeout(unsigned requests, std::chrono::seconds delay) { @@ -391,16 +392,16 @@ TEST_F(RPCClientTestApi, CoroSelectTimeout) { [&] { return server.CloseQRRequestsCount() >= kCorCount * kQueriesCount; }); EXPECT_EQ(server.CloseQRRequestsCount(), kCorCount * kQueriesCount); err = rx.AddNamespace(reindexer::NamespaceDef(kNamespaceName + std::to_string(index))); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); finished[index] = true; }); } loop.run(); for (size_t i = 0; i < kCorCount; ++i) { - EXPECT_TRUE(finished[i]); + ASSERT_TRUE(finished[i]); } Error const err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroRequestCancels) { @@ -412,7 +413,7 @@ TEST_F(RPCClientTestApi, CoroRequestCancels) { loop.spawn([&loop, &finished] { reindexer::client::CoroReindexer rx; auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db", loop); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); { CancelRdxContext ctx; @@ -440,7 +441,7 @@ TEST_F(RPCClientTestApi, CoroRequestCancels) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroSuccessfullRequestWithTimeout) { @@ -455,15 +456,15 @@ TEST_F(RPCClientTestApi, CoroSuccessfullRequestWithTimeout) { config.RequestTimeout = seconds(6); reindexer::client::CoroReindexer rx(config); auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db", loop); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); finished = true; }); loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroErrorLoginResponse) { @@ -483,7 +484,7 @@ TEST_F(RPCClientTestApi, CoroErrorLoginResponse) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroStatus) { @@ -504,7 +505,7 @@ TEST_F(RPCClientTestApi, CoroStatus) { err = rx.Status(); ASSERT_TRUE(err.ok()) << err.what(); err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); loop.sleep(std::chrono::milliseconds(20)); // Allow reading coroutine to handle disconnect err = rx.Status(); ASSERT_EQ(err.code(), errNetwork) << err.what(); @@ -621,8 +622,7 @@ TEST_F(RPCClientTestApi, CoroUpserts) { for (auto& it : qr) { ASSERT_TRUE(it.Status().ok()) << it.Status().what(); } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); finished = true; }); @@ -713,8 +713,7 @@ TEST_F(RPCClientTestApi, ServerRestart) { ready = true; wg.wait(); - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); finished = true; }); @@ -728,7 +727,7 @@ TEST_F(RPCClientTestApi, ServerRestart) { // Shutdown server step = Step::ShutdownInProgress; Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); step = Step::ShutdownDone; std::this_thread::sleep_for(std::chrono::milliseconds(300)); @@ -861,8 +860,7 @@ TEST_F(RPCClientTestApi, CoroUpdatesFilteringByNs) { ASSERT_TRUE(reciever2.AwaitNamespaces(0)); ASSERT_TRUE(reciever3.AwaitNamespaces(0)); } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); err = rx.Status(); ASSERT_FALSE(err.ok()) << err.what(); @@ -964,8 +962,7 @@ TEST_F(RPCClientTestApi, FetchingWithJoin) { EXPECT_EQ(ser.Slice(), expected); i++; } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); }); loop.run(); @@ -1044,8 +1041,7 @@ TEST_F(RPCClientTestApi, AggregationsFetching) { } } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); }); loop.run(); diff --git a/cpp_src/gtests/tests/unit/rtree_test.cc b/cpp_src/gtests/tests/unit/rtree_test.cc index 5a5bf7eed..578bef44d 100644 --- a/cpp_src/gtests/tests/unit/rtree_test.cc +++ b/cpp_src/gtests/tests/unit/rtree_test.cc @@ -7,7 +7,7 @@ #include "core/index/rtree/rstarsplitter.h" #include "gtest/gtest.h" #include "reindexer_api.h" -#include "tools/random.h" +#include "tools/randompoint.h" namespace { @@ -72,7 +72,7 @@ static void TestInsert() { size_t insertedCount = 0; for (size_t i = 0; i < 10000; ++i) { - const auto p = randPoint(kRange); + const auto p = reindexer::randPoint(kRange); const auto insertRes{tree.insert(reindexer::Point{p})}; if (insertRes.second) { ++insertedCount; @@ -100,7 +100,7 @@ static void TestIterators() { size_t dublicates = 0; for (size_t i = 0; i < 10000 + dublicates; ++i) { - const auto res = tree.insert(randPoint(kRange)); + const auto res = tree.insert(reindexer::randPoint(kRange)); if (!res.second) ++dublicates; ASSERT_TRUE(tree.Check()); auto it = tree.begin(), end = tree.end(); @@ -129,6 +129,8 @@ TEST(RTree, RStarIterators) { TestIterators(); } template