Merge branch 'arabic' into 'develop'

Add new languages support for FT #1464

See merge request itv-backend/reindexer!1409

force-build-all

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.18.0"
+const ReindexerVersion = "v3.19.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

bindings/cproto/cproto.go

@@ -19,7 +19,8 @@ import (
 const (
 	defConnPoolSize        = 8
 	defConnPoolLBAlgorithm = bindings.LBRoundRobin
-	pingerTimeoutSec       = 60
+	pingerTimeoutSec       = uint32(60)
+	pingResponseTimeoutSec = uint32(20)
 	defAppName             = "Go-connector"
 
 	opRd = 0
@@ -586,7 +587,7 @@ func (binding *NetCProto) rpcCallNoResults(ctx context.Context, op int, cmd int,
 func (binding *NetCProto) pinger() {
 	timeout := time.Second
 	ticker := time.NewTicker(timeout)
-	var ticksCount uint16
+	var ticksCount uint32
 	for now := range ticker.C {
 		ticksCount++
 		select {
@@ -597,15 +598,32 @@ func (binding *NetCProto) pinger() {
 		if ticksCount == pingerTimeoutSec {
 			ticksCount = 0
 			conns := binding.getAllConns()
+			var wg sync.WaitGroup
+			cmpl := func(buf bindings.RawBuffer, err error) {
+				wg.Done()
+				if buf != nil {
+					buf.Free()
+				}
+			}
 			for _, conn := range conns {
 				if conn.hasError() {
 					continue
 				}
-				if conn.lastReadTime().Add(timeout).Before(now) {
-					buf, _ := conn.rpcCall(context.TODO(), cmdPing, uint32(binding.timeouts.RequestTimeout*time.Second))
-					buf.Free()
+				if !conn.lastReadTime().Add(timeout).Before(now) {
+					continue
 				}
+				if cap(conn.seqs)-len(conn.seqs) > 0 {
+					continue
+				}
+				timeout := pingResponseTimeoutSec
+				if uint32(binding.timeouts.RequestTimeout) > timeout {
+					timeout = uint32(binding.timeouts.RequestTimeout)
+				}
+
+				wg.Add(1)
+				conn.rpcCallAsync(context.TODO(), cmdPing, uint32(timeout), cmpl)
 			}
+			wg.Wait()
 		}
 	}
 }

changelog.md

@@ -1,3 +1,51 @@
+# Version 3.19.0 (XX.09.2023)
+## Core
+- [fea] Added background namespaces deleteion. Previously in some cases atomic transactions could cause the namespace's deletion in the querie's execution thread, which lead to the latency spike
+- [fea] Added locks and flush timings for UPDATE and DELETE queries into [slow log](readme.md#slow-actions-logging)
+- [fea] Added support for the [COUNT/COUNT_CACHED](readme.md#aggregations) aggregations for the MERGE-queries
+- [fea] Added basic support for the SUM/MAX/MIN aggregations for the MERGE-queries without LIMIT/OFFSET
+- [fea] Added more information about JOINs condtitions injection into `explain` (check `on_conditions_injections` field in the explain results)
+- [fea] Added support for store (`-`) UUID indexes
+- [fea] Optimized string<->UUID convertions
+- [fix] Fixed `total_us` time calculation in the `explain` (previously `preselect` time was in the explain results, but was not counted in the `total_us` field)
+- [fix] Fixed JOINs conditions injection for the queries with multiple JOINs. Previously this optimization could skip some of the JOIN queries
+- [fix] Fixed multiple UPDATE arrays concatenation issues: empty arrays, string array, array with scalar, etc.
+- [fix] Composite indexes over the other composite indexes were explicitly disabled
+- [fix] Fixed check for nested JOINs and MERGEs (nested JOIN/MERGE is not allowed)
+- [fix] Multiple JSON-paths were explicitly disabled for the scalar indexes (multiple JSON-paths are allowed for composite and array indexes only)
+- [fix] Fixed crash on the CJSON deserialization for array indexes with multiple JSON-paths
+
+## Replication
+- [fix] Fixed handling of the updates buffer overflow (previously this logic may lead to multiple extra resyncs)
+
+## Fulltext
+- [fea] Added support for Armenian, Hebrew, Arabic and Devanagari alphabets (prefixes/postfixes/suffixes search, typos, stop-words and synonyms are supported)
+
+## Reindexer server
+- [fix] Fixed SSE4.2 support check on startup (server has to print error message if SSE support is required for the current build)
+- [fix] Fixed DELETE-queries compatibility between `reindexer_server v3` and `reindexer_tool v4` in cproto mode
+- [fix] Fixed RPC-timeout handling (extra check for unread data wasd added). Previously connect could be dropped in case of the some very long queries
+
+## Go connector
+- [fea] Optimized string<->UUID convertions
+- [fix] Fixed transactions `Commit`/`Rollback` stucking in case of the incomatible items' sctructs in client and DB
+- [fix] Fixed `builtinserver` logger initialization in cases when single app run multiple instances of the `builtinserver`
+- [fix] Fixed RPC-connections background pinging (Ping requests became asynchronous)
+
+## Build
+- [fix] Fixed build with GCC 13.2
+
+## Face
+- [fea] Changed the numeric values position in the Grid
+- [fea] Added the ability to use spaces for JSON paths
+- [fix] Fixed the Statistics chart for undefined replication.wal_size
+- [fix] Fixed the column set for the namespace items during the namespace switching
+- [fix] Fixed the JSON paths view for values included spaces
+- [fix] Changed the value format for width on the integer for sqlquery
+- [fix] Fixed the bug related to the query history on the Namespace Items list
+- [fix] Fixed the column titles in the table settings menu on the Performance page
+- [fix] Added the validation of the negative values for the index settings
+
 # Version 3.18.0 (29.07.2023)
 ## Core
 - [fea] Increased max indexes count for each namespace up to 255 user-defined indexes (previously it was 63)

cjson/creflect.go

@@ -3,7 +3,6 @@ package cjson
 import (
 	"fmt"
 	"reflect"
-	"strings"
 	"unsafe"
 
 	"github.com/restream/reindexer/v3/bindings"
@@ -107,17 +106,44 @@ func (pl *payloadIface) ptr(field, idx, typ int) unsafe.Pointer {
 const hexChars = "0123456789abcdef"
 
 func createUuid(v [2]uint64) string {
-	var b strings.Builder
-	b.Grow(36)
-	for i, j := 0, 0; i < 36; i++ {
-		switch i {
-		case 8, 13, 18, 23: b.WriteByte('-')
-		default:
-			b.WriteByte(hexChars[(v[j / 16] >> ((15 - j % 16) * 4)) & 0xF])
-			j++
-		}
-	}
-	return b.String()
+	buf := make([]byte, 36)
+	buf[0] = hexChars[(v[0] >> 60) & 0xF];
+	buf[1] = hexChars[(v[0] >> 56) & 0xF];
+	buf[2] = hexChars[(v[0] >> 52) & 0xF];
+	buf[3] = hexChars[(v[0] >> 48) & 0xF];
+	buf[4] = hexChars[(v[0] >> 44) & 0xF];
+	buf[5] = hexChars[(v[0] >> 40) & 0xF];
+	buf[6] = hexChars[(v[0] >> 36) & 0xF];
+	buf[7] = hexChars[(v[0] >> 32) & 0xF];
+	buf[8] = '-';
+	buf[9] = hexChars[(v[0] >> 28) & 0xF];
+	buf[10] = hexChars[(v[0] >> 24) & 0xF];
+	buf[11] = hexChars[(v[0] >> 20) & 0xF];
+	buf[12] = hexChars[(v[0] >> 16) & 0xF];
+	buf[13] = '-';
+	buf[14] = hexChars[(v[0] >> 12) & 0xF];
+	buf[15] = hexChars[(v[0] >> 8) & 0xF];
+	buf[16] = hexChars[(v[0] >> 4) & 0xF];
+	buf[17] = hexChars[v[0] & 0xF];
+	buf[18] = '-';
+	buf[19] = hexChars[(v[1] >> 60) & 0xF];
+	buf[20] = hexChars[(v[1] >> 56) & 0xF];
+	buf[21] = hexChars[(v[1] >> 52) & 0xF];
+	buf[22] = hexChars[(v[1] >> 48) & 0xF];
+	buf[23] = '-';
+	buf[24] = hexChars[(v[1] >> 44) & 0xF];
+	buf[25] = hexChars[(v[1] >> 40) & 0xF];
+	buf[26] = hexChars[(v[1] >> 36) & 0xF];
+	buf[27] = hexChars[(v[1] >> 32) & 0xF];
+	buf[28] = hexChars[(v[1] >> 28) & 0xF];
+	buf[29] = hexChars[(v[1] >> 24) & 0xF];
+	buf[30] = hexChars[(v[1] >> 20) & 0xF];
+	buf[31] = hexChars[(v[1] >> 16) & 0xF];
+	buf[32] = hexChars[(v[1] >> 12) & 0xF];
+	buf[33] = hexChars[(v[1] >> 8) & 0xF];
+	buf[34] = hexChars[(v[1] >> 4) & 0xF];
+	buf[35] = hexChars[v[1] & 0xF];
+	return string(buf)
 }
 
 func (pl *payloadIface) getInt(field, idx int) int {