[test]

Restream · Dec 13, 2024 · 16bf25a · 16bf25a
1 parent d2d4679
commit 16bf25a
Show file tree

Hide file tree

Showing 250 changed files with 244,307 additions and 2,184 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -2,61 +2,10 @@ name: build
 
 on: push
 jobs:
-  build-windows-latest:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        compiler: ["Visual Studio 17 2022"]
-      fail-fast: false
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Build Reindexer
-        run: |
-          mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" .. -Ax64
-          cmake --build . --config Release
-          cmake --build . --config Release --target face
-          cmake --build . --config Release --target swagger
-          cpack
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: windows-latest
-          path: build/*.exe
-          if-no-files-found: error
-
-  build-windows-2019:
-    runs-on: windows-2019
-    strategy:
-      matrix:
-        compiler: ["Visual Studio 16 2019", "MinGW Makefiles"]
-      fail-fast: false
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Build Reindexer
-        run: |
-          mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" ..
-          cmake --build . --config Release
-          cmake --build . --config Release --target face
-          cmake --build . --config Release --target swagger
-          cpack
-
   build:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
-        include:
-          - os: ubuntu-latest
-            sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
-          - os: ubuntu-latest
-            sanitizer: TSAN
-            cc: gcc-12
-            cxx: g++-12
+        os: [ubuntu-22.04]
       fail-fast: false
     runs-on: ${{matrix.os}}
     env:
@@ -108,93 +57,13 @@ jobs:
           path: artifacts.tar
           if-no-files-found: error
 
-  test:
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
-        test: ['C++', 'GO']
-        include:
-          - os: ubuntu-latest
-            sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
-            test: 'C++'
-          - os: ubuntu-latest
-            sanitizer: ASAN
-            test: 'GO'
-#         - os: ubuntu-latest
-#           sanitizer: TSAN
-#           cc: gcc-12
-#           cxx: g++-12
-#           test: 'C++'
-          - os: ubuntu-latest
-            sanitizer: TSAN
-            test: 'GO'
-      fail-fast: false
-    runs-on: ${{matrix.os}}
-    needs: build
-    if: always()
-    env:
-      CC: ${{matrix.cc}}
-      CXX: ${{matrix.cxx}}
-      TEST: ${{matrix.test}}
-      SANITIZER: ${{matrix.sanitizer}}
-    steps:
-      - name: Checkout repository
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
-        uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v5
-        if: ${{ matrix.os == 'macos-13' && matrix.test == 'GO' }}
-        with:
-          go-version: '1.22.x'
-          check-latest: true
-      - name: Download ${{matrix.os}}${{matrix.sanitizer}} Artifacts
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{matrix.os}}${{matrix.sanitizer}}
-      - name: 'Untar Artifacts'
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
-        run: tar -xvf artifacts.tar
-      - name: Prepare Environment
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
-        env:
-          OS: ${{matrix.os}}
-        run: |
-          if [[ $OS == ubuntu* ]]; then
-            sudo ./dependencies.sh
-          else
-            ./dependencies.sh
-          fi
-          if [[ $TEST == 'GO' ]]; then
-            go mod download
-          elif [[ $OS == ubuntu* ]]; then
-            ./.github/workflows/install_gtest_parallel.sh
-          fi
-      - name: Tests
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
-        run: |
-          if [[ $TEST == 'GO' ]]; then
-            if [[ $SANITIZER == 'ASAN' ]]; then
-              export REINDEXER_GH_CI_ASAN=true
-            fi
-            if [[ -z "$SANITIZER" ]]; then
-              go test -timeout 15m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000
-            else
-              go test -timeout 35m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000
-            fi
-          else
-            cd build
-            ctest --verbose
-          fi
-
   test-pyreindexer:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-22.04]
       fail-fast: false
     runs-on: ${{matrix.os}}
+    timeout-minutes: 10
     needs: build
     if: always()
     env:
@@ -210,7 +79,7 @@ jobs:
         run: |
           if [[ $OS == ubuntu* ]]; then
             sudo ./dependencies.sh
-            python3 -m pip install setuptools
+            python3 -m pip install setuptools build
           else
             ./dependencies.sh
           fi
@@ -239,7 +108,9 @@ jobs:
         with:
           repository: restream/reindexer-py
       - name: Install PyReindexer
-        run: sudo python3 setup.py install
+        run: |
+          python -m build
+          python -m pip install .
       - name: Test PyReindexer
         run: |
           cd pyreindexer

diff --git a/appveyor.yml b/appveyor.yml
@@ -12,11 +12,11 @@ environment:
       TOOLCHAIN: x86_64-8.1.0-posix-seh-rt_v6-rev0
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
-    - BUILD_TYPE: Release
-      COMPILER: MinGW
-      PLATFORM: Win32
-      TOOLCHAIN: i686-8.1.0-posix-dwarf-rt_v6-rev0
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+#    - BUILD_TYPE: Release
+#      COMPILER: MinGW
+#      PLATFORM: Win32
+#      TOOLCHAIN: i686-8.1.0-posix-dwarf-rt_v6-rev0
+#      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
 build_script:
   - git describe --tags

diff --git a/bindings.go b/bindings.go
@@ -2,6 +2,7 @@ package reindexer
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -555,6 +556,11 @@ func WithStrictJoinHandlers() interface{} {
 	return bindings.OptionStrictJoinHandlers{EnableStrictJoinHandlers: true}
 }
 
+// Enables connection to Reindexer using TLS. If tls.Config is nil TLS is disabled
+func WithTLSConfig(config *tls.Config) interface{} {
+	return bindings.OptionTLS{Config: config}
+}
+
 // WithReconnectionStrategy allows to configure the behavior during reconnect after error.
 // Strategy used for reconnect to server on connection error
 // AllowUnknownNodes allows to add dsn from cluster node, that was not set in client dsn list

diff --git a/bindings/consts.go b/bindings/consts.go
@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v4.18.0"
+const ReindexerVersion = "v4.19.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (
@@ -200,5 +200,4 @@ const (
 const (
 	ShardingNotSet   = -1
 	ShardingProxyOff = -2
-	NotSharded       = -3
 )
diff --git a/bindings/cproto/connection.go b/bindings/cproto/connection.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -142,6 +143,8 @@ type connectionImpl struct {
 	requestDedicatedThread bool
 	loggerOwner            LoggerOwner
 	eventsHandler          bindings.EventsHandler
+
+	configTLS *tls.Config
 }
 
 type newConnParams struct {
@@ -154,6 +157,7 @@ type newConnParams struct {
 	requestDedicatedThread bool
 	caps                   bindings.BindingCapabilities
 	envetsHandler          bindings.EventsHandler
+	tls                    bindings.OptionTLS
 }
 
 func newConnection(
@@ -175,6 +179,7 @@ func newConnection(
 		requestDedicatedThread: params.requestDedicatedThread,
 		loggerOwner:            loggerOwner,
 		eventsHandler:          eventsHandler,
+		configTLS:              params.tls.Config,
 	}
 	for i := 0; i < queueSize; i++ {
 		c.seqs <- uint32(i)
@@ -259,15 +264,21 @@ func (c *connectionImpl) deadlineTicker() {
 }
 
 func (c *connectionImpl) connect(ctx context.Context, dsn *url.URL) (err error) {
-	var d net.Dialer
-	if dsn.Scheme == "cproto" {
-		if c.conn, err = d.DialContext(ctx, "tcp", dsn.Host); err != nil {
+	var netDialer net.Dialer
+	if dsn.Scheme == "cprotos" {
+		tlsDialer := tls.Dialer{Config: c.configTLS, NetDialer: &netDialer}
+		if c.conn, err = tlsDialer.DialContext(ctx, "tcp", dsn.Host); err != nil {
+			return err
+		}
+		c.conn.(*tls.Conn).NetConn().(*net.TCPConn).SetNoDelay(true)
+	} else if dsn.Scheme == "cproto" {
+		if c.conn, err = netDialer.DialContext(ctx, "tcp", dsn.Host); err != nil {
 			return err
 		}
 		c.conn.(*net.TCPConn).SetNoDelay(true)
 	} else {
-		d.LocalAddr = nil
-		if c.conn, err = d.DialContext(ctx, "unix", dsn.Host); err != nil {
+		netDialer.LocalAddr = nil
+		if c.conn, err = netDialer.DialContext(ctx, "unix", dsn.Host); err != nil {
 			return err
 		}
 	}

diff --git a/bindings/cproto/cproto.go b/bindings/cproto/cproto.go
@@ -60,6 +60,7 @@ var emptyLogger bindings.NullLogger
 func init() {
 	rand.Seed(time.Now().UnixNano())
 	bindings.RegisterBinding("cproto", new(NetCProto))
+	bindings.RegisterBinding("cprotos", new(NetCProto))
 	if runtime.GOOS != "windows" {
 		bindings.RegisterBinding("ucproto", new(NetCProto))
 	}
@@ -88,6 +89,7 @@ type NetCProto struct {
 	lock               sync.RWMutex
 	logger             bindings.Logger
 	logMtx             sync.RWMutex
+	tls                bindings.OptionTLS
 }
 
 type dsn struct {
@@ -310,6 +312,8 @@ func (binding *NetCProto) Init(u []url.URL, eh bindings.EventsHandler, options .
 		case bindings.OptionReconnectionStrategy:
 			binding.dsn.reconnectionStrategy = v.Strategy
 			binding.dsn.allowUnknownNodes = v.AllowUnknownNodes
+		case bindings.OptionTLS:
+			binding.tls = v
 
 		default:
 			fmt.Printf("Unknown cproto option: %#v\n", option)
@@ -396,7 +400,8 @@ func (binding *NetCProto) createConnParams() newConnParams {
 		appName:                binding.appName,
 		enableCompression:      binding.compression.EnableCompression,
 		requestDedicatedThread: binding.dedicatedThreads.DedicatedThreads,
-		caps:                   binding.caps}
+		caps:                   binding.caps,
+		tls:                    binding.tls}
 }
 
 func (binding *NetCProto) createEventConn(ctx context.Context, connParams newConnParams, eventsSubOptsJSON []byte) error {

diff --git a/bindings/interface.go b/bindings/interface.go
@@ -2,6 +2,7 @@ package bindings
 
 import (
 	"context"
+	"crypto/tls"
 	"net/url"
 	"time"
 
@@ -384,6 +385,10 @@ type OptionReconnectionStrategy struct {
 	AllowUnknownNodes bool
 }
 
+type OptionTLS struct {
+	Config *tls.Config
+}
+
 type Status struct {
 	Err     error
 	CProto  StatusCProto

diff --git a/changelog.md b/changelog.md
@@ -1,3 +1,31 @@
+# Version 4.19.0 (XX.12.2024)
+
+## Core
+- [fea] Added automatic masking for reindexer user's credentials in log files and replication/sharding JSON's
+
+## Replication
+- [fea] Optimized memory footprint for online-updates queue
+
+## Reindexer server
+- [fea] Added `OpenSSL` support for HTTP-server (`https`) and RPC-server (`cprotos`). [Read more...](cpp_src/readme.md#tls-support)
+- [fea] Added special user roles `sharding` and `replication` with all required rights and restrictions for correspondig scenarios
+- [fea] Added support for SHA256/SHA512-based encription for user's passwords in `users.yml`. [Read more...](cpp_src/readme.md#authentication)
+- [fea] Added HTTP method `GET api/v1/user/role` to check current user's role
+- [fix] Fixed possible update buffer overflow for [events subscription](readme.md#events-subscription) in TCP-mode
+
+## Ported
+- [fea/fix] Ported all the fixes and features from [v3.30.0](https://github.com/Restream/reindexer/releases/tag/v3.30.0)
+
+## Face
+- [fea] Added `column_size` field to the Memory table
+- [fea] Added `splitter` field to the Full text index config
+- [fix] Fixed wrong info message about disabled statistics when the database didn't have Namespaces
+- [fix] Fixed last row position in the Memory table
+- [fix] Fixed wrong input field clearing on cache settings page
+- [fix] Fixed sort order icon on the tables
+- [fix] Fixed Replication Statistics table behavior on tabs switching
+- [fix] Fixed data saving in JSON view on the Fulltext index settings page
+
 # Version 4.18.0 (31.10.2024) *beta*
 ## Replication
 - [fix] Fixed possible origin LSN missmatch in snapshots during WAL/Force syncs