From dd250137266aa965d8859b899bbe4ae1b8ac00cf Mon Sep 17 00:00:00 2001 From: reindexer-bot <@> Date: Tue, 24 Sep 2024 22:33:34 +0000 Subject: [PATCH] Merge branch 'bugfix/1798-qr-items-pb-serialization' into 'develop' #1798 QueryResults items Protobuf serialization See merge request itv-backend/reindexer!1678 --- clang-tidy/.clang-tidy | 31 - clang-tidy/.clang-tidy-ignore | 10 - clang-tidy/run-clang-tidy-18.py | 429 -------------- cpp_src/CMakeLists.txt | 7 +- .../test/test_storage_compatibility.sh | 197 +++++++ .../cmd/reindexer_tool/commandsexecutor.cc | 16 +- cpp_src/core/cjson/baseencoder.cc | 7 +- cpp_src/core/cjson/baseencoder.h | 9 +- cpp_src/core/cjson/cjsondecoder.cc | 1 - cpp_src/core/cjson/cjsondecoder.h | 24 +- cpp_src/core/cjson/defaultvaluecoder.cc | 171 ------ cpp_src/core/cjson/defaultvaluecoder.h | 41 -- cpp_src/core/cjson/jsondecoder.cc | 7 +- cpp_src/core/cjson/jsondecoder.h | 8 +- cpp_src/core/cjson/msgpackdecoder.cc | 6 +- cpp_src/core/cjson/protobufdecoder.cc | 13 +- cpp_src/core/cjson/uuid_recoders.h | 18 +- cpp_src/core/keyvalue/variant.cc | 16 +- cpp_src/core/namespace/itemsloader.cc | 6 +- cpp_src/core/namespace/namespaceimpl.cc | 93 +-- cpp_src/core/namespace/namespaceimpl.h | 1 - cpp_src/core/queryresults/queryresults.cc | 8 + cpp_src/gtests/bench/ft_bench.cc | 11 +- cpp_src/gtests/bench/reindexer_bench.cc | 11 +- cpp_src/gtests/tests/API/base_tests.cc | 6 +- cpp_src/gtests/tests/fixtures/ft_api.cc | 2 +- cpp_src/gtests/tests/fixtures/get_pk_api.h | 2 +- .../gtests/tests/fixtures/storage_lazy_load.h | 2 +- .../tests/unit/ft/ft_incremental_build.cc | 2 +- cpp_src/gtests/tests/unit/index_tuple_test.cc | 529 ------------------ cpp_src/gtests/tests/unit/protobuf_test.cc | 6 +- cpp_src/net/cproto/dispatcher.h | 8 +- cpp_src/net/iserverconnection.h | 2 +- cpp_src/server/grpc/reindexerservice.cc | 2 - cpp_src/server/httpserver.cc | 5 - test/queries_test.go | 4 +- 36 files changed, 315 insertions(+), 1396 deletions(-) delete mode 100644 clang-tidy/.clang-tidy delete mode 100644 clang-tidy/.clang-tidy-ignore delete mode 100755 clang-tidy/run-clang-tidy-18.py create mode 100755 cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh delete mode 100644 cpp_src/core/cjson/defaultvaluecoder.cc delete mode 100644 cpp_src/core/cjson/defaultvaluecoder.h delete mode 100644 cpp_src/gtests/tests/unit/index_tuple_test.cc diff --git a/clang-tidy/.clang-tidy b/clang-tidy/.clang-tidy deleted file mode 100644 index 063df74f1..000000000 --- a/clang-tidy/.clang-tidy +++ /dev/null @@ -1,31 +0,0 @@ -Checks: 'clang-diagnostic-*, - clang-analyzer-*, - performance-*, - bugprone-*, - -bugprone-exception-escape, - -bugprone-branch-clone, - -bugprone-easily-swappable-parameters, - -bugprone-macro-parentheses, - -bugprone-signed-char-misuse, - -bugprone-narrowing-conversions, - -bugprone-reserved-identifier, - -bugprone-implicit-widening-of-multiplication-result, - -bugprone-assignment-in-if-condition, - -bugprone-parent-virtual-call, - -bugprone-integer-division, - -bugprone-unhandled-self-assignment, - -bugprone-inc-dec-in-conditions, - -clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling, - -performance-no-int-to-ptr, - -performance-enum-size, - -performance-avoid-endl' -# clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling - too many unnecessary warning in vendored code -# performance-no-int-to-ptr - consider how to fix this -# bugprone-macro-parentheses - consider fixing -WarningsAsErrors: '*' -HeaderFilterRegex: '.*(?= 4.0.0 are given under - # the top level key 'Diagnostics' in the output yaml files - mergekey = "Diagnostics" - merged=[] - for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): - content = yaml.safe_load(open(replacefile, 'r')) - if not content: - continue # Skip empty files. - merged.extend(content.get(mergekey, [])) - - if merged: - # MainSourceFile: The key is required by the definition inside - # include/clang/Tooling/ReplacementsYaml.h, but the value - # is actually never used inside clang-apply-replacements, - # so we set it to '' here. - output = {'MainSourceFile': '', mergekey: merged} - with open(mergefile, 'w') as out: - yaml.safe_dump(output, out) - else: - # Empty the file: - open(mergefile, 'w').close() - - -def find_binary(arg, name, build_path): - """Get the path for a binary or exit""" - if arg: - if shutil.which(arg): - return arg - else: - raise SystemExit( - "error: passed binary '{}' was not found or is not executable" - .format(arg)) - - built_path = os.path.join(build_path, "bin", name) - binary = shutil.which(name) or shutil.which(built_path) - if binary: - return binary - else: - raise SystemExit( - "error: failed to find {} in $PATH or at {}" - .format(name, built_path)) - - -def apply_fixes(args, clang_apply_replacements_binary, tmpdir): - """Calls clang-apply-fixes on a given directory.""" - invocation = [clang_apply_replacements_binary] - invocation.append('-ignore-insert-conflict') - if args.format: - invocation.append('-format') - if args.style: - invocation.append('-style=' + args.style) - invocation.append(tmpdir) - subprocess.call(invocation) - - -def run_tidy(args, clang_tidy_binary, tmpdir, build_path, queue, lock, - failed_files): - """Takes filenames out of queue and runs clang-tidy on them.""" - while True: - name = queue.get() - invocation = get_tidy_invocation(name, clang_tidy_binary, args.checks, - tmpdir, build_path, args.header_filter, - args.allow_enabling_alpha_checkers, - args.extra_arg, args.extra_arg_before, - args.quiet, args.config_file, args.config, - args.line_filter, args.use_color, - args.plugins) - - proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, err = proc.communicate() - if proc.returncode != 0: - if proc.returncode < 0: - msg = "%s: terminated by signal %d\n" % (name, -proc.returncode) - err += msg.encode('utf-8') - failed_files.append(name) - with lock: - sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8')) - if len(err) > 0: - sys.stdout.flush() - sys.stderr.write(err.decode('utf-8')) - queue.task_done() - - -def main(): - parser = argparse.ArgumentParser(description='Runs clang-tidy over all files ' - 'in a compilation database. Requires ' - 'clang-tidy and clang-apply-replacements in ' - '$PATH or in your build directory.') - parser.add_argument('-allow-enabling-alpha-checkers', - action='store_true', help='allow alpha checkers from ' - 'clang-analyzer.') - parser.add_argument('-clang-tidy-binary', metavar='PATH', - default='clang-tidy-18', - help='path to clang-tidy binary') - parser.add_argument('-clang-apply-replacements-binary', metavar='PATH', - default='clang-apply-replacements-18', - help='path to clang-apply-replacements binary') - parser.add_argument('-checks', default=None, - help='checks filter, when not specified, use clang-tidy ' - 'default') - config_group = parser.add_mutually_exclusive_group() - config_group.add_argument('-config', default=None, - help='Specifies a configuration in YAML/JSON format: ' - ' -config="{Checks: \'*\', ' - ' CheckOptions: {x: y}}" ' - 'When the value is empty, clang-tidy will ' - 'attempt to find a file named .clang-tidy for ' - 'each source file in its parent directories.') - config_group.add_argument('-config-file', default=None, - help='Specify the path of .clang-tidy or custom config ' - 'file: e.g. -config-file=/some/path/myTidyConfigFile. ' - 'This option internally works exactly the same way as ' - '-config option after reading specified config file. ' - 'Use either -config-file or -config, not both.') - parser.add_argument('-header-filter', default=None, - help='regular expression matching the names of the ' - 'headers to output diagnostics from. Diagnostics from ' - 'the main file of each translation unit are always ' - 'displayed.') - parser.add_argument('-line-filter', default=None, - help='List of files with line ranges to filter the' - 'warnings.') - if yaml: - parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes', - help='Create a yaml file to store suggested fixes in, ' - 'which can be applied with clang-apply-replacements.') - parser.add_argument('-j', type=int, default=0, - help='number of tidy instances to be run in parallel.') - parser.add_argument('files', nargs='*', default=['.*'], - help='files to be processed (regex on path)') - parser.add_argument('-fix', action='store_true', help='apply fix-its') - parser.add_argument('-format', action='store_true', help='Reformat code ' - 'after applying fixes') - parser.add_argument('-style', default='file', help='The style of reformat ' - 'code after applying fixes') - parser.add_argument('-use-color', type=strtobool, nargs='?', const=True, - help='Use colors in diagnostics, overriding clang-tidy\'s' - ' default behavior. This option overrides the \'UseColor' - '\' option in .clang-tidy file, if any.') - parser.add_argument('-p', dest='build_path', - help='Path used to read a compile command database.') - parser.add_argument('-extra-arg', dest='extra_arg', - action='append', default=[], - help='Additional argument to append to the compiler ' - 'command line.') - parser.add_argument('-extra-arg-before', dest='extra_arg_before', - action='append', default=[], - help='Additional argument to prepend to the compiler ' - 'command line.') - parser.add_argument('-ignore', default=DEFAULT_CLANG_TIDY_IGNORE, - help='File path to clang-tidy-ignore') - parser.add_argument('-quiet', action='store_true', - help='Run clang-tidy in quiet mode') - parser.add_argument('-load', dest='plugins', - action='append', default=[], - help='Load the specified plugin in clang-tidy.') - args = parser.parse_args() - - db_path = 'compile_commands.json' - - if args.build_path is not None: - build_path = args.build_path - else: - # Find our database - build_path = find_compilation_database(db_path) - - clang_tidy_binary = find_binary(args.clang_tidy_binary, "clang-tidy", - build_path) - - tmpdir = None - if args.fix or (yaml and args.export_fixes): - clang_apply_replacements_binary = find_binary( - args.clang_apply_replacements_binary, "clang-apply-replacements", - build_path) - tmpdir = tempfile.mkdtemp() - - try: - invocation = get_tidy_invocation("", clang_tidy_binary, args.checks, - None, build_path, args.header_filter, - args.allow_enabling_alpha_checkers, - args.extra_arg, args.extra_arg_before, - args.quiet, args.config_file, args.config, - args.line_filter, args.use_color, - args.plugins) - invocation.append('-list-checks') - invocation.append('-') - if args.quiet: - # Even with -quiet we still want to check if we can call clang-tidy. - with open(os.devnull, 'w') as dev_null: - subprocess.check_call(invocation, stdout=dev_null) - else: - subprocess.check_call(invocation) - except: - print("Unable to run clang-tidy.", file=sys.stderr) - sys.exit(1) - - # Load the database and extract all files. - database = json.load(open(os.path.join(build_path, db_path))) - files = set([make_absolute(entry['file'], entry['directory']) - for entry in database]) - files, excluded = filter_files(args.ignore, files) - if excluded: - print("Excluding the following files:\n" + "\n".join(excluded) + "\n") - - max_task = args.j - if max_task == 0: - max_task = multiprocessing.cpu_count() - - # Build up a big regexy filter from all command line arguments. - file_name_re = re.compile('|'.join(args.files)) - - return_code = 0 - try: - # Spin up a bunch of tidy-launching threads. - task_queue = queue.Queue(max_task) - # List of files with a non-zero return code. - failed_files = [] - lock = threading.Lock() - for _ in range(max_task): - t = threading.Thread(target=run_tidy, - args=(args, clang_tidy_binary, tmpdir, build_path, - task_queue, lock, failed_files)) - t.daemon = True - t.start() - - # Fill the queue with files. - for name in files: - if file_name_re.search(name): - task_queue.put(name) - - # Wait for all threads to be done. - task_queue.join() - if len(failed_files): - return_code = 1 - - except KeyboardInterrupt: - # This is a sad hack. Unfortunately subprocess goes - # bonkers with ctrl-c and we start forking merrily. - print('\nCtrl-C detected, goodbye.') - if tmpdir: - shutil.rmtree(tmpdir) - os.kill(0, 9) - - if yaml and args.export_fixes: - print('Writing fixes to ' + args.export_fixes + ' ...') - try: - merge_replacement_files(tmpdir, args.export_fixes) - except: - print('Error exporting fixes.\n', file=sys.stderr) - traceback.print_exc() - return_code=1 - - if args.fix: - print('Applying fixes ...') - try: - apply_fixes(args, clang_apply_replacements_binary, tmpdir) - except: - print('Error applying fixes.\n', file=sys.stderr) - traceback.print_exc() - return_code = 1 - - if tmpdir: - shutil.rmtree(tmpdir) - sys.exit(return_code) - - -if __name__ == '__main__': - main() diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index 8b287338f..f40d1fe2e 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -87,7 +87,7 @@ endif() if(MSVC) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -MP -MD") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -wd4800 -wd4396 -wd4503 -MP -MD") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4244 -wd4267 -wd4996 -wd4717 -wd4800 -wd4396 -wd4503 -MP -MD /bigobj") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -SAFESEH:NO") else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wswitch-enum") @@ -619,8 +619,3 @@ add_custom_target(collect_coverage COMMAND genhtml coverage_filtered.info -o coverage_output COMMENT "Collecting Reindexer coverage" ) - -# Configure compile options extra -if(MSVC) - target_compile_options(${TARGET} PRIVATE /bigobj) -endif() diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100755 index 000000000..873181e20 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,197 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +echo "## choose latest release rpm file" +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 3) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(python3 cpp_src/cmd/reindexer_server/test/get_last_rx_version.py -v 4) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://github.com/restream/reindexer_testdata/-/raw/main/dump_demo.zip" --output dump_demo.zip; +unzip -o dump_demo.zip # unzips into demo_test.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"name":"account_recommendations","replication":{"data_hash":6833710705,"data_count":1}}, +{"name":"admin_roles","replication":{"data_hash":1896088071,"data_count":2}}, +{"name":"asset_video_servers","replication":{"data_hash":7404222244,"data_count":97}}, +{"name":"banners","replication":{"data_hash":0,"data_count":0}}, +{"name":"channels","replication":{"data_hash":457292509431319,"data_count":3941}}, +{"name":"child_account_recommendations","replication":{"data_hash":6252344969,"data_count":1}}, +{"name":"collections","replication":{"data_hash":0,"data_count":0}}, +{"name":"epg","replication":{"data_hash":-7049751653258,"data_count":1623116}}, +{"name":"epg_genres","replication":{"data_hash":8373644068,"data_count":1315}}, +{"name":"karaoke_items","replication":{"data_hash":5858155773472,"data_count":4500}}, +{"name":"media_item_recommendations","replication":{"data_hash":-6520334670,"data_count":35886}}, +{"name":"media_items","replication":{"data_hash":-1824301168479972392,"data_count":65448}}, +{"name":"media_view_templates","replication":{"data_hash":0,"data_count":0}}, +{"name":"menu_items","replication":{"data_hash":0,"data_count":0}}, +{"name":"purchase_options_ext_dict","replication":{"data_hash":24651210926,"data_count":3}}, +{"name":"radio_channels","replication":{"data_hash":37734732881,"data_count":28}}, +{"name":"recom_epg_archive_default","replication":{"data_hash":0,"data_count":0}}, +{"name":"recom_epg_archive_personal","replication":{"data_hash":0,"data_count":0}}, +{"name":"recom_epg_live_default","replication":{"data_hash":0,"data_count":0}}, +{"name":"recom_epg_live_personal","replication":{"data_hash":0,"data_count":0}}, +{"name":"recom_media_items_default","replication":{"data_hash":8288213744,"data_count":3}}, +{"name":"recom_media_items_personal","replication":{"data_hash":0,"data_count":0}}, +{"name":"recom_media_items_similars","replication":{"data_hash":-672103903,"data_count":33538}}, +{"name":"services","replication":{"data_hash":0,"data_count":0}}, +{"name":"wp_imports_tasks","replication":{"data_hash":777859741066,"data_count":1145}}, +{"name":"wp_tasks_schedule","replication":{"data_hash":12595790956,"data_count":4}}, +{"name":"wp_tasks_tasks","replication":{"data_hash":28692716680,"data_count":281}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f demo_test.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select name, replication.data_hash, replication.data_count from #memstats order by name'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select name, replication.data_hash, replication.data_count from #memstats order by name'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f demo_test.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select name, replication.data_hash, replication.data_count from #memstats order by name'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select name, replication.data_hash, replication.data_count from #memstats order by name'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc index 085741e3c..254242b65 100644 --- a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc @@ -25,7 +25,6 @@ const std::string kVariableOutput = "output"; const std::string kOutputModeJson = "json"; const std::string kOutputModeTable = "table"; const std::string kOutputModePretty = "pretty"; -const std::string kOutputModePrettyCollapsed = "collapsed"; const std::string kBenchNamespace = "rxtool_bench"; const std::string kBenchIndex = "id"; @@ -157,7 +156,10 @@ Error CommandsExecutor::fromFileImpl(std::istream& in) { while (GetStatus().running && std::getline(in, line.str)) { if (reindexer::checkIfStartsWith("\\upsert ", line.str) || reindexer::checkIfStartsWith("\\delete ", line.str)) { try { - cmdCh.push(line); + LineData l; + l.lineNum = line.lineNum; + reindexer::deepCopy(l.str, line.str); + cmdCh.push(std::move(l)); } catch (std::exception&) { break; } @@ -506,7 +508,7 @@ Error CommandsExecutor::processImpl(const std::string& command) noe } catch (std::exception& e) { return Error(errLogic, "std::exception during command's execution: %s", e.what()); } catch (...) { - return Error(errLogic, "Unknow exception during command's execution"); + return Error(errLogic, "Unknown exception during command's execution"); } } return Error(errParams, "Unknown command '%s'. Type '\\help' to list of available commands", token); @@ -683,7 +685,7 @@ Error CommandsExecutor::commandUpsert(const std::string& command) { status = db().Upsert(nsName, item); if (!fromFile_ && status.ok()) { - output_() << "Upserted successfuly: 1 items" << std::endl; + output_() << "Upserted successfully: 1 items" << std::endl; } return status; } @@ -809,7 +811,7 @@ Error CommandsExecutor::commandDump(const std::string& command) { } for (auto it : itemResults) { - if (auto err = it.Status(); !err.ok()) { + if (err = it.Status(); !err.ok()) { return err; } if (cancelCtx_.IsCancelled()) { @@ -1103,7 +1105,7 @@ Error CommandsExecutor::commandProcessDatabase err = db().Status(); } if (err.ok()) { - output_() << "Succesfully connected to " << currentDsn << std::endl; + output_() << "Successfully connected to " << currentDsn << std::endl; } return err; } else if (subCommand == "create"sv) { @@ -1119,7 +1121,7 @@ Error CommandsExecutor::commandProcessDatabase std::vector dbNames; err = db().EnumDatabases(dbNames); if (std::find(dbNames.begin(), dbNames.end(), std::string(dbName)) != dbNames.end()) { - output_() << "Succesfully created database '" << dbName << "'" << std::endl; + output_() << "Successfully created database '" << dbName << "'" << std::endl; } else { std::cerr << "Error on database '" << dbName << "' creation" << std::endl; } diff --git a/cpp_src/core/cjson/baseencoder.cc b/cpp_src/core/cjson/baseencoder.cc index 992a268a6..b3950919d 100644 --- a/cpp_src/core/cjson/baseencoder.cc +++ b/cpp_src/core/cjson/baseencoder.cc @@ -1,6 +1,5 @@ #include "baseencoder.h" #include -#include #include "cjsonbuilder.h" #include "cjsontools.h" #include "core/keyvalue/p_string.h" @@ -42,7 +41,7 @@ void BaseEncoder::Encode(ConstPayload& pl, Builder& builder, IAdditiona } objectScalarIndexes_.reset(); - std::fill_n(std::begin(fieldsoutcnt_), pl.NumFields(), 0); + std::fill(fieldsoutcnt_.begin(), fieldsoutcnt_.end(), 0); builder.SetTagsMatcher(tagsMatcher_); if constexpr (kWithTagsPathTracking) { builder.SetTagsPath(&curTagsPath_); @@ -167,11 +166,11 @@ bool BaseEncoder::encode(ConstPayload* pl, Serializer& rdser, Builder& [&](KeyValueType::String) { builder.Array(tagName, pl->GetArray(tagField).subspan(cnt, count), cnt); }, [&](KeyValueType::Uuid) { builder.Array(tagName, pl->GetArray(tagField).subspan(cnt, count), cnt); }, [](OneOf) noexcept { - assertrx(0); + assertrx(false); abort(); }); } - cnt += count; + cnt += int(count); break; } case TAG_NULL: diff --git a/cpp_src/core/cjson/baseencoder.h b/cpp_src/core/cjson/baseencoder.h index 9eafa4808..bb1cd818d 100644 --- a/cpp_src/core/cjson/baseencoder.h +++ b/cpp_src/core/cjson/baseencoder.h @@ -10,7 +10,6 @@ namespace reindexer { class TagsMatcher; -class JsonBuilder; class MsgPackBuilder; class ProtobufBuilder; class CsvBuilder; @@ -38,7 +37,7 @@ class IAdditionalDatasource { template class BaseEncoder { public: - BaseEncoder(const TagsMatcher* tagsMatcher, const FieldsSet* filter = nullptr); + explicit BaseEncoder(const TagsMatcher* tagsMatcher, const FieldsSet* filter = nullptr); void Encode(ConstPayload& pl, Builder& builder, IAdditionalDatasource* = nullptr); void Encode(std::string_view tuple, Builder& wrSer, IAdditionalDatasource*); @@ -61,9 +60,9 @@ class BaseEncoder { std::string_view getPlTuple(ConstPayload& pl); - const TagsMatcher* tagsMatcher_; - std::array fieldsoutcnt_{0}; - const FieldsSet* filter_; + const TagsMatcher* tagsMatcher_{nullptr}; + std::array fieldsoutcnt_; + const FieldsSet* filter_{nullptr}; WrSerializer tmpPlTuple_; TagsPath curTagsPath_; IndexedTagsPathInternalT indexedTagsPath_; diff --git a/cpp_src/core/cjson/cjsondecoder.cc b/cpp_src/core/cjson/cjsondecoder.cc index 26c7d8549..3064f14e8 100644 --- a/cpp_src/core/cjson/cjsondecoder.cc +++ b/cpp_src/core/cjson/cjsondecoder.cc @@ -11,7 +11,6 @@ bool CJsonDecoder::decodeCJson(Payload& pl, Serializer& rdser, WrSerializer& wrs const ctag tag = rdser.GetCTag(); TagType tagType = tag.Type(); if (tagType == TAG_END) { - recoder.Serialize(wrser); wrser.PutCTag(kCTagEnd); return false; } diff --git a/cpp_src/core/cjson/cjsondecoder.h b/cpp_src/core/cjson/cjsondecoder.h index 443fea4c7..747c2fd7e 100644 --- a/cpp_src/core/cjson/cjsondecoder.h +++ b/cpp_src/core/cjson/cjsondecoder.h @@ -17,10 +17,8 @@ class Recoder { [[nodiscard]] virtual TagType Type(TagType oldTagType) = 0; virtual void Recode(Serializer&, WrSerializer&) const = 0; virtual void Recode(Serializer&, Payload&, int tagName, WrSerializer&) = 0; - [[nodiscard]] virtual bool Match(int field) noexcept = 0; - [[nodiscard]] virtual bool Match(TagType, const TagsPath&) = 0; - virtual void Serialize(WrSerializer& wrser) = 0; - virtual bool Reset() = 0; + [[nodiscard]] virtual bool Match(int field) const noexcept = 0; + [[nodiscard]] virtual bool Match(const TagsPath&) const = 0; virtual ~Recoder() = default; }; @@ -49,7 +47,7 @@ class CJsonDecoder { class IndexedSkipFilter { public: - IndexedSkipFilter(const FieldsSet& f) noexcept : f_(&f) {} + explicit IndexedSkipFilter(const FieldsSet& f) noexcept : f_(&f) {} IndexedSkipFilter MakeCleanCopy() const noexcept { return IndexedSkipFilter(*f_); } IndexedSkipFilter MakeSkipFilter() const noexcept { return IndexedSkipFilter(*f_); } @@ -57,7 +55,7 @@ class CJsonDecoder { RX_ALWAYS_INLINE bool match(const TagsPath&) const noexcept { return false; } private: - const FieldsSet* f_; + const FieldsSet* f_{nullptr}; }; class RestrictingFilter { @@ -85,8 +83,8 @@ class CJsonDecoder { } private: - const FieldsSet* f_; - bool match_; + const FieldsSet* f_{nullptr}; + bool match_{false}; }; class DummyRecoder { @@ -96,7 +94,6 @@ class CJsonDecoder { RX_ALWAYS_INLINE bool Recode(Serializer&, Payload&, int, WrSerializer&) const noexcept { return false; } RX_ALWAYS_INLINE TagType RegisterTagType(TagType tagType, int) const noexcept { return tagType; } RX_ALWAYS_INLINE TagType RegisterTagType(TagType tagType, const TagsPath&) const noexcept { return tagType; } - RX_ALWAYS_INLINE void Serialize(WrSerializer&) const {} }; class DefaultRecoder { public: @@ -121,14 +118,13 @@ class CJsonDecoder { return needToRecode_ ? r_->Type(tagType) : tagType; } RX_ALWAYS_INLINE TagType RegisterTagType(TagType tagType, const TagsPath& tagsPath) { - needToRecode_ = r_->Match(tagType, tagsPath); + needToRecode_ = r_->Match(tagsPath); return needToRecode_ ? r_->Type(tagType) : tagType; } - RX_ALWAYS_INLINE void Serialize(WrSerializer& wser) const { r_->Serialize(wser); } private: - Recoder* r_; - bool needToRecode_; + Recoder* r_{nullptr}; + bool needToRecode_{false}; }; struct NamedTagOpt {}; struct NamelessTagOpt {}; @@ -166,7 +162,7 @@ class CJsonDecoder { TagsMatcher& tagsMatcher_; TagsPath tagsPath_; - int32_t arrayLevel_ = 0; + int32_t arrayLevel_{0}; ScalarIndexesSetT objectScalarIndexes_; // storage for owning strings obtained from numbers std::deque& storage_; diff --git a/cpp_src/core/cjson/defaultvaluecoder.cc b/cpp_src/core/cjson/defaultvaluecoder.cc deleted file mode 100644 index d33713150..000000000 --- a/cpp_src/core/cjson/defaultvaluecoder.cc +++ /dev/null @@ -1,171 +0,0 @@ -#include "defaultvaluecoder.h" - -namespace reindexer { - -DefaultValueCoder::DefaultValueCoder(std::string_view ns, const PayloadFieldType& fld, std::vector&& tps, int16_t fieldIdx) - : ns_(ns), - field_(fld.Name()), - tags_(std::move(tps)), - fieldIdx_(fieldIdx), - type_(fld.Type().ToTagType()), - array_(fld.IsArray()), - basePath_(&tags_.front()) {} - -bool DefaultValueCoder::Match(int field) noexcept { - // non-nested field present in tuple - if ((field == fieldIdx_) && ready()) { - state_ = State::found; - } - return false; // returned result is always same -} - -bool DefaultValueCoder::Match(TagType tt, const TagsPath& tp) { - static const bool result = false; // returned result is always same - - // nothing to look for (start tuple global object) - if (tp.empty()) { - state_ = State::wait; - // inArray_ = false; - // arrField_ = 0; - return result; - } - - // found\recorded earlier - if ((state_ == State::found) || ((state_ == State::write) /* && !inArray_*/)) { - return result; - } - - // check if active array has been processed - const bool arrayTag = (tt == TAG_ARRAY); - // if (inArray_) { - // inArray_ = ((tt == TAG_OBJECT) || arrayTag) ? (tp.back() == arrField_) : (tp[tp.size() - 2] == arrField_); // -2 pre-last item - // // recorded earlier - stop it - // if (!inArray_ && (state_ == State::write)) { - // return result; - // } - // } - - // try match nested field - if (tt == TAG_OBJECT) { - assertrx(state_ != State::found); - match(tp); - return result; - } - - // may be end element of adjacent nested field - if (arrayTag) { - if (tp.size() <= basePath_->size() && std::equal(tp.begin(), tp.end(), basePath_->begin())) { - // Do not create anything inside objects arrays #1819 - state_ = State::found; - return result; - } - // inArray_ = (tp.front() == basePath_->front()); - // arrField_ = tp.back(); - } - - // not nested - if (copyPos_ == 0) { - return result; - } - - // detect array insertion into array (not supported) - if (arrayTag && array_) { - state_ = State::found; // do nothing - } else if ((tp.front() == basePath_->front()) && (tp.size() > basePath_->size())) { - ++nestingLevel_; - } - - return result; -} - -void DefaultValueCoder::Serialize(WrSerializer& wrser) { - if (blocked()) { - return; // skip processing - } - - // skip nested levels - if ((basePath_->size() > 1) || (nestingLevel_ > 1)) { - assertrx(nestingLevel_ > 0); - --nestingLevel_; - - // new field - move to valid level - if (nestingLevel_ > copyPos_) { - return; - } - } - - const auto written = write(wrser); - Reset(); - state_ = written ? State::write : State::found; -} - -bool DefaultValueCoder::Reset() noexcept { - nestingLevel_ = 1; - copyPos_ = 0; - // NOTE: return true when updating tuple - return (state_ == State::write); -} - -void DefaultValueCoder::match(const TagsPath& tp) { - ++nestingLevel_; - - for (auto& path : tags_) { - if (path.front() != tp.front()) { - continue; - } - - copyPos_ = 1; - auto pathSize = path.size(); - auto sz = std::min(pathSize, tp.size()); - for (size_t idx = 1; idx < sz; ++idx) { - if (path[idx] != tp[idx]) { - break; - } - ++copyPos_; - - // we are trying to add field with non-nested paths, but an intersection was found in additional nested paths. - // Stop, throw an error - if (tags_.front().size() == 1) { - throw Error(errLogic, "Cannot add field with name '%s' to namespace '%s'. One of nested json paths is already in use", - field_, ns_); - } - } - state_ = State::match; - basePath_ = &path; - break; - } -} - -bool DefaultValueCoder::write(WrSerializer& wrser) const { - if (array_ && copyPos_ + 1 < basePath_->size()) { - // Do not create multiple levels for nested array indexes to avoid problems with decoding in Go/Java connectors. #1819 - return false; - } - int32_t nestedObjects = 0; - for (size_t idx = copyPos_, sz = basePath_->size(); idx < sz; ++idx) { - auto tagName = (*basePath_)[idx]; - // real index field in last tag - const bool finalTag = (idx == (sz - 1)); - if (finalTag) { - if (array_) { - wrser.PutCTag(ctag{TAG_ARRAY, tagName, fieldIdx_}); - wrser.PutVarUint(0); - } else { - wrser.PutCTag(ctag{type_, tagName, fieldIdx_}); - } - break; - } - - // start nested object - wrser.PutCTag(ctag{TAG_OBJECT, tagName}); - ++nestedObjects; - } - - // add end tags to all objects - while (nestedObjects-- > 0) { - wrser.PutCTag(kCTagEnd); - } - return true; -} - -} // namespace reindexer diff --git a/cpp_src/core/cjson/defaultvaluecoder.h b/cpp_src/core/cjson/defaultvaluecoder.h deleted file mode 100644 index 3f4276dc2..000000000 --- a/cpp_src/core/cjson/defaultvaluecoder.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include "cjsondecoder.h" - -namespace reindexer { - -class DefaultValueCoder : public Recoder { -public: - DefaultValueCoder(std::string_view ns, const PayloadFieldType& fld, std::vector&& tps, int16_t fieldIdx); - RX_ALWAYS_INLINE TagType Type(TagType tt) noexcept override final { return tt; } - [[nodiscard]] bool Match(int f) noexcept override final; - [[nodiscard]] bool Match(TagType tt, const TagsPath& tp) override final; - RX_ALWAYS_INLINE void Recode(Serializer&, WrSerializer&) const noexcept override final { assertrx(false); } - RX_ALWAYS_INLINE void Recode(Serializer&, Payload&, int, WrSerializer&) noexcept override final { assertrx(false); } - void Serialize(WrSerializer& wrser) override final; - bool Reset() noexcept override final; - -private: - void match(const TagsPath& tp); - [[nodiscard]] bool write(WrSerializer& wrser) const; - [[nodiscard]] RX_ALWAYS_INLINE bool blocked() const noexcept { return ((state_ == State::found) || (state_ == State::write)); } - [[nodiscard]] RX_ALWAYS_INLINE bool ready() const noexcept { return ((state_ == State::wait) || (state_ == State::match)); } - -private: - const std::string ns_; - const std::string field_; - const std::vector tags_; - const int16_t fieldIdx_{0}; - const TagType type_; - const bool array_{false}; - - const TagsPath* basePath_{nullptr}; - enum class State { wait, found, match, write } state_{State::wait}; - uint32_t nestingLevel_{1}; - uint32_t copyPos_{0}; - - // bool inArray_{false}; - // int16_t arrField_{0}; -}; - -} // namespace reindexer diff --git a/cpp_src/core/cjson/jsondecoder.cc b/cpp_src/core/cjson/jsondecoder.cc index 85d3ac79d..7ee9a0e0e 100644 --- a/cpp_src/core/cjson/jsondecoder.cc +++ b/cpp_src/core/cjson/jsondecoder.cc @@ -115,13 +115,8 @@ void JsonDecoder::decodeJson(Payload* pl, CJsonBuilder& builder, const gason::Js builder.Null(tagName); break; case gason::JSON_ARRAY: { - ObjType type; CounterGuardIR32 g(arrayLevel_); - if (gason::isHomogeneousArray(v)) { - type = ObjType::TypeArray; - } else { - type = ObjType::TypeObjectArray; - } + ObjType type = (gason::isHomogeneousArray(v)) ? ObjType::TypeArray : ObjType::TypeObjectArray; auto arrNode = builder.Array(tagName, type); for (const auto& elem : v) { decodeJson(pl, arrNode, elem.value, 0, match); diff --git a/cpp_src/core/cjson/jsondecoder.h b/cpp_src/core/cjson/jsondecoder.h index 73dad50f2..066f6740a 100644 --- a/cpp_src/core/cjson/jsondecoder.h +++ b/cpp_src/core/cjson/jsondecoder.h @@ -8,8 +8,8 @@ namespace reindexer { class JsonDecoder { public: - explicit JsonDecoder(TagsMatcher& tagsMatcher) noexcept : tagsMatcher_(tagsMatcher), filter_(nullptr) {} - JsonDecoder(TagsMatcher& tagsMatcher, const FieldsSet* filter) noexcept : tagsMatcher_(tagsMatcher), filter_(filter) {} + explicit JsonDecoder(TagsMatcher& tagsMatcher, const FieldsSet* filter = nullptr) noexcept + : tagsMatcher_(tagsMatcher), filter_(filter) {} Error Decode(Payload& pl, WrSerializer& wrSer, const gason::JsonValue& v); void Decode(std::string_view json, CJsonBuilder& builder, const TagsPath& fieldPath); @@ -21,8 +21,8 @@ class JsonDecoder { TagsMatcher& tagsMatcher_; TagsPath tagsPath_; - const FieldsSet* filter_; - int32_t arrayLevel_ = 0; + const FieldsSet* filter_{nullptr}; + int32_t arrayLevel_{0}; ScalarIndexesSetT objectScalarIndexes_; }; diff --git a/cpp_src/core/cjson/msgpackdecoder.cc b/cpp_src/core/cjson/msgpackdecoder.cc index e08e55ab5..8b72c9f21 100644 --- a/cpp_src/core/cjson/msgpackdecoder.cc +++ b/cpp_src/core/cjson/msgpackdecoder.cc @@ -145,13 +145,13 @@ Error MsgPackDecoder::Decode(std::string_view buf, Payload& pl, WrSerializer& wr } catch (const Error& err) { return err; } catch (const std::exception& ex) { - return Error(errNotValid, "%s", ex.what()); + return {errNotValid, "%s", ex.what()}; } catch (...) { // all internal errors shall be handled and converted to Error - return Error(errNotValid, "Unexpected exception"); + return {errNotValid, "Unexpected exception"}; } - return Error(); + return {}; } constexpr std::string_view ToString(msgpack_object_type type) { diff --git a/cpp_src/core/cjson/protobufdecoder.cc b/cpp_src/core/cjson/protobufdecoder.cc index a8313ab01..7d2dcc04a 100644 --- a/cpp_src/core/cjson/protobufdecoder.cc +++ b/cpp_src/core/cjson/protobufdecoder.cc @@ -1,14 +1,13 @@ #include "protobufdecoder.h" #include "core/schema.h" #include "estl/protobufparser.h" -#include "protobufbuilder.h" namespace reindexer { void ArraysStorage::UpdateArraySize(int tagName, int field) { GetArray(tagName, field); } CJsonBuilder& ArraysStorage::GetArray(int tagName, int field) { - assertrx(indexes_.size() > 0); + assertrx(!indexes_.empty()); auto it = data_.find(tagName); if (it == data_.end()) { indexes_.back().emplace_back(tagName); @@ -25,7 +24,7 @@ CJsonBuilder& ArraysStorage::GetArray(int tagName, int field) { void ArraysStorage::onAddObject() { indexes_.emplace_back(h_vector()); } void ArraysStorage::onObjectBuilt(CJsonBuilder& parent) { - assertrx(indexes_.size() > 0); + assertrx(!indexes_.empty()); for (int tagName : indexes_.back()) { auto it = data_.find(tagName); assertrx(it != data_.end()); @@ -69,10 +68,10 @@ void ProtobufDecoder::setValue(Payload& pl, CJsonBuilder& builder, ProtobufValue Error ProtobufDecoder::decodeArray(Payload& pl, CJsonBuilder& builder, const ProtobufValue& item) { ProtobufObject object(item.As(), *schema_, tagsPath_, tm_); ProtobufParser parser(object); - bool packed = item.IsOfPrimitiveType(); - int field = tm_.tags2field(tagsPath_.data(), tagsPath_.size()); + const bool packed = item.IsOfPrimitiveType(); + const int field = tm_.tags2field(tagsPath_.data(), tagsPath_.size()); if (field > 0) { - auto& f = pl.Type().Field(field); + const auto& f = pl.Type().Field(field); if rx_unlikely (!f.IsArray()) { throw Error(errLogic, "Error parsing protobuf field '%s' - got array, expected scalar %s", f.Name(), f.Type().Name()); } @@ -104,7 +103,7 @@ Error ProtobufDecoder::decodeArray(Payload& pl, CJsonBuilder& builder, const Pro } } } - return Error(); + return {}; } Error ProtobufDecoder::decode(Payload& pl, CJsonBuilder& builder, const ProtobufValue& item) { diff --git a/cpp_src/core/cjson/uuid_recoders.h b/cpp_src/core/cjson/uuid_recoders.h index cdc264ac3..9bb51c701 100644 --- a/cpp_src/core/cjson/uuid_recoders.h +++ b/cpp_src/core/cjson/uuid_recoders.h @@ -19,10 +19,8 @@ class RecoderUuidToString : public Recoder { } void Recode(Serializer&, WrSerializer&) const override final; void Recode(Serializer&, Payload&, int, WrSerializer&) override final { assertrx(false); } - [[nodiscard]] bool Match(int) noexcept override final { return false; } - [[nodiscard]] bool Match(TagType, const TagsPath& tp) noexcept override final { return tagsPath_ == tp; } - void Serialize(WrSerializer&) override final {} - bool Reset() override final { return false; } + [[nodiscard]] bool Match(int) const noexcept final { return false; } + [[nodiscard]] bool Match(const TagsPath& tp) const noexcept final { return tagsPath_ == tp; } private: TagsPath tagsPath_; @@ -54,8 +52,8 @@ class RecoderStringToUuidArray : public Recoder { } return TAG_ARRAY; } - [[nodiscard]] bool Match(int f) noexcept override final { return f == field_; } - [[nodiscard]] bool Match(TagType, const TagsPath&) noexcept override final { return false; } + [[nodiscard]] bool Match(int f) const noexcept final { return f == field_; } + [[nodiscard]] bool Match(const TagsPath&) const noexcept final { return false; } void Recode(Serializer&, WrSerializer&) const override final { assertrx(false); } void Recode(Serializer& rdser, Payload& pl, int tagName, WrSerializer& wrser) override final { if (fromNotArrayField_) { @@ -78,8 +76,6 @@ class RecoderStringToUuidArray : public Recoder { wrser.PutVarUint(count); } } - void Serialize(WrSerializer&) override final {} - bool Reset() override final { return false; } private: const int field_{std::numeric_limits::max()}; @@ -98,15 +94,13 @@ class RecoderStringToUuid : public Recoder { } return TAG_UUID; } - [[nodiscard]] bool Match(int f) noexcept override final { return f == field_; } - [[nodiscard]] bool Match(TagType, const TagsPath&) noexcept override final { return false; } + [[nodiscard]] bool Match(int f) const noexcept final { return f == field_; } + [[nodiscard]] bool Match(const TagsPath&) const noexcept final { return false; } void Recode(Serializer&, WrSerializer&) const override final { assertrx(false); } void Recode(Serializer& rdser, Payload& pl, int tagName, WrSerializer& wrser) override final { pl.Set(field_, Variant{rdser.GetStrUuid()}, true); wrser.PutCTag(ctag{TAG_UUID, tagName, field_}); } - void Serialize(WrSerializer&) override final {} - bool Reset() override final { return false; } private: const int field_{std::numeric_limits::max()}; diff --git a/cpp_src/core/keyvalue/variant.cc b/cpp_src/core/keyvalue/variant.cc index 5005ef5dd..9e645d481 100644 --- a/cpp_src/core/keyvalue/variant.cc +++ b/cpp_src/core/keyvalue/variant.cc @@ -520,31 +520,31 @@ class Comparator { return compare(v1_.As(), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Bool, KeyValueType::Int) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(int(v1_.As()), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Bool, KeyValueType::Int64) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(int64_t(v1_.As()), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Bool, KeyValueType::Double) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(double(v1_.As()), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Int, KeyValueType::Bool) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(v1_.As(), int(v2_.As())); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Int, KeyValueType::Int) const noexcept { return compare(v1_.As(), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Int, KeyValueType::Int64) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(int64_t(v1_.As()), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Int, KeyValueType::Double) const noexcept { return compare(v1_.As(), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Int64, KeyValueType::Bool) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(v1_.As(), int64_t(v2_.As())); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Int64, KeyValueType::Int) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(v1_.As(), int64_t(v2_.As())); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Int64, KeyValueType::Int64) const noexcept { return compare(v1_.As(), v2_.As()); @@ -553,7 +553,7 @@ class Comparator { return compare(v1_.As(), v2_.As()); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Double, KeyValueType::Bool) const noexcept { - return compare(v1_.As(), v2_.As()); + return compare(v1_.As(), double(v2_.As())); } RX_ALWAYS_INLINE ComparationResult operator()(KeyValueType::Double, KeyValueType::Int) const noexcept { return compare(v1_.As(), v2_.As()); diff --git a/cpp_src/core/namespace/itemsloader.cc b/cpp_src/core/namespace/itemsloader.cc index 1a7f59b44..89795271b 100644 --- a/cpp_src/core/namespace/itemsloader.cc +++ b/cpp_src/core/namespace/itemsloader.cc @@ -57,7 +57,7 @@ void ItemsLoader::reading() { throw Error(errLogic, "Can't load data storage of '%s' - there are no PK fields in ns", ns_.name_); } if (dataSlice.size() < sizeof(int64_t)) { - lastErr = Error(errParseBin, "Not enougth data in data slice"); + lastErr = Error(errParseBin, "Not enough data in data slice"); logPrintf(LogTrace, "Error load item to '%s' from storage: '%s'", ns_.name_, lastErr.what()); ++errCount; continue; @@ -66,7 +66,7 @@ void ItemsLoader::reading() { // Read LSN int64_t lsn = *reinterpret_cast(dataSlice.data()); if (lsn < 0) { - lastErr = Error(errParseBin, "Ivalid LSN value: %d", lsn); + lastErr = Error(errParseBin, "Invalid LSN value: %d", lsn); logPrintf(LogTrace, "Error load item to '%s' from storage: '%s'", ns_.name_, lastErr.what()); ++errCount; continue; @@ -114,7 +114,7 @@ void ItemsLoader::reading() { continue; } item.impl.Value().SetLSN(int64_t(l)); - // Prealloc payload here, because reading|parsing thread is faster then index insertion thread + // Preallocate payload here, because reading|parsing thread is faster than index insertion thread item.preallocPl = PayloadValue(item.impl.GetConstPayload().RealSize()); lck.lock(); diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index f32901545..b6236109a 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -3,7 +3,6 @@ #include #include #include "core/cjson/cjsondecoder.h" -// #include "core/cjson/defaultvaluecoder.h" #include "core/cjson/jsonbuilder.h" #include "core/cjson/uuid_recoders.h" #include "core/formatters/lsn_fmt.h" @@ -99,7 +98,7 @@ NamespaceImpl::NamespaceImpl(const NamespaceImpl& src, AsyncStorage::FullLockT& optimizationState_{NotOptimized}, strHolder_{makeStringsHolder()}, nsUpdateSortedContextMemory_{0}, - dbDestroyed_(false) { + dbDestroyed_{false} { for (auto& idxIt : src.indexes_) { indexes_.push_back(idxIt->Clone()); } @@ -111,23 +110,23 @@ NamespaceImpl::NamespaceImpl(const NamespaceImpl& src, AsyncStorage::FullLockT& NamespaceImpl::NamespaceImpl(const std::string& name, UpdatesObservers& observers) : intrusive_atomic_rc_base(), - indexes_(*this), - name_(name), - payloadType_(name), - tagsMatcher_(payloadType_), - enablePerfCounters_(false), - queryCountCache_( - std::make_unique(config_.cacheConfig.queryCountCacheSize, config_.cacheConfig.queryCountHitsToCache)), - joinCache_(std::make_unique(config_.cacheConfig.joinCacheSize, config_.cacheConfig.joinHitsToCache)), + indexes_{*this}, + name_{name}, + payloadType_{name}, + tagsMatcher_{payloadType_}, + enablePerfCounters_{false}, + queryCountCache_{ + std::make_unique(config_.cacheConfig.queryCountCacheSize, config_.cacheConfig.queryCountHitsToCache)}, + joinCache_{std::make_unique(config_.cacheConfig.joinCacheSize, config_.cacheConfig.joinHitsToCache)}, wal_(getWalSize(config_)), - observers_(&observers), + observers_{&observers}, lastSelectTime_{0}, cancelCommitCnt_{0}, lastUpdateTime_{0}, - nsIsLoading_(false), - serverIdChanged_(false), + nsIsLoading_{false}, + serverIdChanged_{false}, strHolder_{makeStringsHolder()}, - dbDestroyed_(false) { + dbDestroyed_{false} { logPrintf(LogTrace, "NamespaceImpl::NamespaceImpl (%s)", name_); FlagGuardT nsLoadingGuard(nsIsLoading_); items_.reserve(10000); @@ -392,7 +391,7 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes final : private RollBackB private: NamespaceImpl& ns_; std::vector> indexes_; - size_t startIdx_; + size_t startIdx_{0}; }; template <> @@ -476,32 +475,11 @@ class NamespaceImpl::RollBack_updateItems final : private RollBackBase { NamespaceImpl& ns_; RollBack_recreateCompositeIndexes rollbacker_recreateCompositeIndexes_; std::vector> items_; - uint64_t dataHash_; - size_t itemsDataSize_; + uint64_t dataHash_{0}; + size_t itemsDataSize_{0}; std::unique_ptr tuple_; }; -std::vector NamespaceImpl::pickJsonPath(const PayloadFieldType& fld) { - const auto& paths = fld.JsonPaths(); - if (fld.IsArray()) { - std::vector result; - result.reserve(paths.size()); - for (const auto& path : paths) { - auto tags = tagsMatcher_.path2tag(path, false); - result.push_back(std::move(tags)); - // first without nested path - always (any, now last one found) - if ((result.size() > 1) && (result.back().size() == 1)) { - std::swap(result.front(), result.back()); - } - } - return result; - } - - assertrx_throw(paths.size() == 1); - auto tags = tagsMatcher_.path2tag(paths.front(), false); - return {std::move(tags)}; -} - template <> class NamespaceImpl::RollBack_updateItems { public: @@ -565,16 +543,6 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con recoder = std::make_unique(changedField); } } - // TODO: This logic must be reenabled after #1353. Now it's potentially unsafe - // else { - // const auto& indexToUpdate = indexes_[changedField]; - // if (!IsComposite(indexToUpdate->Type()) && !indexToUpdate->Opts().IsSparse()) { - // auto tagsNames = pickJsonPath(fld); - // if (!tagsNames.empty()) { - // recoder = std::make_unique(name_, fld, std::move(tagsNames), changedField); - // } - // } - // } } rollbacker.SaveTuple(); @@ -598,7 +566,6 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con ItemImpl oldItem(oldPlType, plCurr, tagsMatcher_); oldItem.Unsafe(true); newItem.FromCJSON(oldItem, recoder.get()); - const bool itemTupleUpdated = recoder && recoder->Reset(); PayloadValue plNew = oldValue.CopyTo(payloadType_, fieldChangeType == FieldChangeType::Add); plNew.SetLSN(plCurr.GetLSN()); @@ -650,17 +617,6 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con plCurr = std::move(plNew); repl_.dataHash ^= Payload(payloadType_, plCurr).GetHash(); itemsDataSize_ += plCurr.GetCapacity() + sizeof(PayloadValue::dataHeader); - - // update data in storage - if (itemTupleUpdated && storage_.IsValid()) { - pk.Reset(); - data.Reset(); - pk << kRxStorageItemPrefix; - Payload(payloadType_, plCurr).SerializeFields(pk, pkFields()); - data.PutUInt64(plCurr.GetLSN()); - newItem.GetCJSON(data); - storage_.Write(pk.Slice(), data.Slice()); - } } markUpdated(IndexOptimization::Partial); @@ -1351,7 +1307,7 @@ int NamespaceImpl::getIndexByName(std::string_view index) const { } int NamespaceImpl::getIndexByNameOrJsonPath(std::string_view index) const { - int idx; + int idx = 0; if (tryGetIndexByName(index, idx)) { return idx; } @@ -1364,7 +1320,7 @@ int NamespaceImpl::getIndexByNameOrJsonPath(std::string_view index) const { } int NamespaceImpl::getScalarIndexByName(std::string_view index) const { - int idx; + int idx = 0; if (tryGetIndexByName(index, idx)) { if (idx < indexes_.firstCompositePos()) { return idx; @@ -1395,7 +1351,7 @@ bool NamespaceImpl::getIndexByNameOrJsonPath(std::string_view name, int& index) } bool NamespaceImpl::getScalarIndexByName(std::string_view name, int& index) const { - int idx; + int idx = 0; if (tryGetIndexByName(name, idx)) { if (idx < indexes_.firstCompositePos()) { index = idx; @@ -1609,7 +1565,7 @@ void NamespaceImpl::doDelete(IdType id) { storage_.Remove(pk.Slice()); // erase last item - int field; + int field = 0; // erase from composite indexes auto indexesCacheCleaner{GetIndexesCacheCleaner()}; @@ -2984,6 +2940,9 @@ void NamespaceImpl::removeExpiredItems(RdxActivityContext* ctx) { qr.AddNamespace(this, true); auto q = Query(name_).Where(index->Name(), CondLt, expirationThreshold); doDelete(q, qr, rdxCtx); + if (qr.Count()) { + logFmt(LogInfo, "{}: {} items were removed: TTL({}) has expired", name_, qr.Count(), index->Name()); + } } tryForceFlush(std::move(wlck)); } @@ -3228,8 +3187,7 @@ void NamespaceImpl::checkApplySlaveUpdate(bool fromReplication) { if (repl_.slaveMode && !repl_.replicatorEnabled) // readOnly { throw Error(errLogic, "Can't modify read only ns '%s'", name_); - } else if (repl_.slaveMode && repl_.replicatorEnabled) // slave - { + } else if (repl_.slaveMode && repl_.replicatorEnabled) { // slave if (!fromReplication) { logPrintf(LogTrace, "[repl:%s]:%d Can't modify slave ns '%s' repl_.slaveMode=%d repl_.replicatorenabled=%d fromReplication=%d", name_, serverId_, name_, repl_.slaveMode, repl_.replicatorEnabled, fromReplication); @@ -3237,8 +3195,7 @@ void NamespaceImpl::checkApplySlaveUpdate(bool fromReplication) { } else if (repl_.status == ReplicationState::Status::Fatal) { throw Error(errLogic, "Can't modify slave ns '%s', ns has fatal replication error: %s", name_, repl_.replError.what()); } - } else if (!repl_.slaveMode && !repl_.replicatorEnabled) // master - { + } else if (!repl_.slaveMode && !repl_.replicatorEnabled) { // master if (fromReplication) { throw Error(errLogic, "Can't modify master ns '%s' from replicator", name_); } else if (repl_.status == ReplicationState::Status::Fatal) { diff --git a/cpp_src/core/namespace/namespaceimpl.h b/cpp_src/core/namespace/namespaceimpl.h index 3f0a75824..2bd31dd74 100644 --- a/cpp_src/core/namespace/namespaceimpl.h +++ b/cpp_src/core/namespace/namespaceimpl.h @@ -488,7 +488,6 @@ class NamespaceImpl final : public intrusive_atomic_rc_base { // NOLINT(*perfor } size_t getWalSize(const NamespaceConfigData& cfg) const noexcept { return isSystem() ? int64_t(1) : std::max(cfg.walSize, int64_t(1)); } void clearNamespaceCaches(); - std::vector pickJsonPath(const PayloadFieldType& fld); PerfStatCounterMT updatePerfCounter_, selectPerfCounter_; std::atomic_bool enablePerfCounters_{false}; diff --git a/cpp_src/core/queryresults/queryresults.cc b/cpp_src/core/queryresults/queryresults.cc index 0a1ee037d..a56fbf265 100644 --- a/cpp_src/core/queryresults/queryresults.cc +++ b/cpp_src/core/queryresults/queryresults.cc @@ -8,6 +8,7 @@ #include "core/namespace/namespace.h" #include "core/namespace/namespaceimpl.h" #include "joinresults.h" +#include "server/outputparameters.h" #include "tools/catch_and_return.h" namespace reindexer { @@ -316,6 +317,9 @@ Error QueryResults::Iterator::GetProtobuf(WrSerializer& wrser, bool withHdrLen) auto& itemRef = qr_->items_[idx_]; assertrx(qr_->ctxs.size() > itemRef.Nsid()); auto& ctx = qr_->ctxs[itemRef.Nsid()]; + if (!ctx.schema_) { + return Error(errParams, "The schema was not found for Protobuf builder"); + } if (itemRef.Value().IsFree()) { return Error(errNotFound, "Item not found"); @@ -324,6 +328,10 @@ Error QueryResults::Iterator::GetProtobuf(WrSerializer& wrser, bool withHdrLen) ConstPayload pl(ctx.type_, itemRef.Value()); ProtobufEncoder encoder(&ctx.tagsMatcher_); ProtobufBuilder builder(&wrser, ObjType::TypePlain, ctx.schema_.get(), const_cast(&ctx.tagsMatcher_)); + + auto item = builder.Object(kProtoQueryResultsFields.at(kParamItems)); + auto ItemImpl = item.Object(ctx.schema_->GetProtobufNsNumber() + 1); + if (withHdrLen) { auto slicePosSaver = wrser.StartSlice(); encoder.Encode(pl, builder); diff --git a/cpp_src/gtests/bench/ft_bench.cc b/cpp_src/gtests/bench/ft_bench.cc index c0bdb667d..4d2b64b6e 100644 --- a/cpp_src/gtests/bench/ft_bench.cc +++ b/cpp_src/gtests/bench/ft_bench.cc @@ -9,8 +9,6 @@ #include "ft_fixture.h" #include "ft_merge_limit.h" -const std::string kStoragePath = "/tmp/reindex/ft_bench_test"; - using std::shared_ptr; using reindexer::Reindexer; @@ -23,15 +21,16 @@ const int kItemsInBenchDataset = 100'000; #endif int main(int argc, char** argv) { - if (reindexer::fs::RmDirAll(kStoragePath) < 0 && errno != ENOENT) { - std::cerr << "Could not clean working dir '" << kStoragePath << "'."; + const auto storagePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/ft_bench_test"); + if (reindexer::fs::RmDirAll(storagePath) < 0 && errno != ENOENT) { + std::cerr << "Could not clean working dir '" << storagePath << "'."; std::cerr << "Reason: " << strerror(errno) << std::endl; return 1; } - shared_ptr DB = std::make_shared(); - auto err = DB->Connect("builtin://" + kStoragePath); + auto DB = std::make_shared(); + auto err = DB->Connect("builtin://" + storagePath); if (!err.ok()) { return err.code(); } diff --git a/cpp_src/gtests/bench/reindexer_bench.cc b/cpp_src/gtests/bench/reindexer_bench.cc index 669055241..1a094ac9b 100644 --- a/cpp_src/gtests/bench/reindexer_bench.cc +++ b/cpp_src/gtests/bench/reindexer_bench.cc @@ -16,8 +16,6 @@ #include "core/reindexer.h" -const std::string kStoragePath = "/tmp/reindex/bench_test"; - using std::shared_ptr; using reindexer::Reindexer; @@ -33,15 +31,16 @@ const int kItemsInComparatorsBenchDataset = 100'000; #endif int main(int argc, char** argv) { - if (reindexer::fs::RmDirAll(kStoragePath) < 0 && errno != ENOENT) { - std::cerr << "Could not clean working dir '" << kStoragePath << "'."; + const auto storagePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/bench_test"); + if (reindexer::fs::RmDirAll(storagePath) < 0 && errno != ENOENT) { + std::cerr << "Could not clean working dir '" << storagePath << "'."; std::cerr << "Reason: " << strerror(errno) << std::endl; return 1; } - shared_ptr DB = std::make_shared(); - auto err = DB->Connect("builtin://" + kStoragePath); + auto DB = std::make_shared(); + auto err = DB->Connect("builtin://" + storagePath); if (!err.ok()) { return err.code(); } diff --git a/cpp_src/gtests/tests/API/base_tests.cc b/cpp_src/gtests/tests/API/base_tests.cc index c75694529..1371ce729 100644 --- a/cpp_src/gtests/tests/API/base_tests.cc +++ b/cpp_src/gtests/tests/API/base_tests.cc @@ -2045,11 +2045,11 @@ TEST_F(ReindexerApi, IntFieldConvertToStringIndexTest) { } TEST_F(ReindexerApi, MetaIndexTest) { - const std::string kStoragePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/meta_index_test/"); - reindexer::fs::RmDirAll(kStoragePath); + const auto storagePath = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex/meta_index_test/"); + reindexer::fs::RmDirAll(storagePath); // ignore result auto rx = std::make_unique(); - auto err = rx->Connect("builtin://" + kStoragePath); + auto err = rx->Connect("builtin://" + storagePath); ASSERT_TRUE(err.ok()) << err.what(); err = rx->OpenNamespace(default_namespace, StorageOpts().Enabled().CreateIfMissing()); diff --git a/cpp_src/gtests/tests/fixtures/ft_api.cc b/cpp_src/gtests/tests/fixtures/ft_api.cc index 8db2d3aff..9df3eee8d 100644 --- a/cpp_src/gtests/tests/fixtures/ft_api.cc +++ b/cpp_src/gtests/tests/fixtures/ft_api.cc @@ -1,7 +1,7 @@ #include "ft_api.h" void FTApi::Init(const reindexer::FtFastConfig& ftCfg, unsigned nses, const std::string& storage) { - rt.reindexer.reset(new reindexer::Reindexer); + rt.reindexer = std::make_shared(); if (!storage.empty()) { auto err = rt.reindexer->Connect("builtin://" + storage); ASSERT_TRUE(err.ok()) << err.what(); diff --git a/cpp_src/gtests/tests/fixtures/get_pk_api.h b/cpp_src/gtests/tests/fixtures/get_pk_api.h index 703a729eb..b5698d220 100644 --- a/cpp_src/gtests/tests/fixtures/get_pk_api.h +++ b/cpp_src/gtests/tests/fixtures/get_pk_api.h @@ -110,7 +110,7 @@ class ExtractPK : public testing::Test { void SetUp() { colors_ = {"red", "green", "blue", "yellow", "purple", "orange"}; names_ = {"bubble", "dog", "tomorrow", "car", "dinner", "dish"}; - db_.reset(new Reindexer); + db_ = std::make_shared(); } Data randomItemData() { diff --git a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h index c68e2c466..754c36f0e 100644 --- a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h +++ b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h @@ -5,7 +5,7 @@ class DISABLED_StorageLazyLoadApi : public ReindexerApi { public: - DISABLED_StorageLazyLoadApi() : pk_(0), inserted_(0) { rt.reindexer.reset(new Reindexer); } + DISABLED_StorageLazyLoadApi() : pk_(0), inserted_(0) { rt.reindexer = std::make_shared(); } ~DISABLED_StorageLazyLoadApi() { dropNs(); } void SetUp() override { diff --git a/cpp_src/gtests/tests/unit/ft/ft_incremental_build.cc b/cpp_src/gtests/tests/unit/ft/ft_incremental_build.cc index 577422741..10b6c9c42 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_incremental_build.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_incremental_build.cc @@ -29,7 +29,7 @@ class FTIncrementalBuildApi : public FTApi { enum class StrictSuffixValidation { No, Yes }; void Init(const reindexer::FtFastConfig& ftCfg) { - rt.reindexer.reset(new reindexer::Reindexer); + rt.reindexer = std::make_shared(); auto err = rt.reindexer->OpenNamespace(GetDefaultNamespace()); ASSERT_TRUE(err.ok()) << err.what(); rt.DefineNamespaceDataset(GetDefaultNamespace(), {IndexDeclaration{"id", "hash", "int", IndexOpts().PK(), 0}, diff --git a/cpp_src/gtests/tests/unit/index_tuple_test.cc b/cpp_src/gtests/tests/unit/index_tuple_test.cc deleted file mode 100644 index ce7729fe2..000000000 --- a/cpp_src/gtests/tests/unit/index_tuple_test.cc +++ /dev/null @@ -1,529 +0,0 @@ -#include -#include "reindexer_api.h" -#include "tools/fsops.h" - -class IndexTupleTest : public ReindexerApi { -public: - void SetUp() override { - reindexer::fs::RmDirAll(kStoragePath); - ReindexerApi::SetUp(); - } - - [[nodiscard]] std::string CreateEmptyNamespace(std::string_view ns) { return createNS(ns, R"json({"id":%d})json"); } - - [[nodiscard]] std::string CreateNamespace(std::string_view ns) { - static const char pattern[] = - R"json({"id":%d,"objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}]})json"; - return createNS(ns, pattern); - } - - [[nodiscard]] std::string CreateSparseNamespace(std::string_view ns) { - return createNS(ns, R"json({"id":%d,"fld1":1,"fld2":{"nested":"test"}})json"); - } - - [[nodiscard]] std::string CreateArrayNamespace(std::string_view ns) { - static const char pattern[] = R"json({"id":%d,"obj":{"val":10},"arr":[1,2,3]})json"; - return createNS(ns, pattern); - } - - void DoTestDefault(const std::shared_ptr& reindexer, std::string_view ns, const reindexer::IndexDef& indexDef, - std::string_view pattern, std::string_view field, const VariantArray& expectedValues, - std::string_view description) const { - auto err = rt.reindexer->AddIndex(ns, indexDef); - ASSERT_TRUE(err.ok()) << err.what() << "\n" << description; - - validateResults(reindexer, ns, pattern, field, expectedValues, description); - } - - void DoTestEmpty(const std::shared_ptr& reindexer, std::string_view ns, const reindexer::IndexDef& indexDef, - std::string_view pattern, std::string_view description) const { - auto err = reindexer->AddIndex(ns, indexDef); - ASSERT_TRUE(err.ok()) << err.what(); - - checkExpectations(reindexer, ns, pattern, description); - } - - void DoCallAndCheckError(const std::shared_ptr& reindexer, std::string_view ns, - const reindexer::IndexDef& indexDef, std::string_view errMsg) const { - std::vector items; - getItems(reindexer, ns, items); - - auto err = reindexer->AddIndex(ns, indexDef); - ASSERT_FALSE(err.ok()); - ASSERT_EQ(err.what(), errMsg); - - checkItems(reindexer, ns, items); - } - - void ValidateReloadState(const std::shared_ptr& reindexer, std::string_view ns, std::string_view pattern, - std::string_view description, const std::string& storagePath) const { - auto err = rt.reindexer->CloseNamespace(ns); - ASSERT_TRUE(err.ok()) << err.what(); - - err = rt.reindexer->OpenNamespace(ns, StorageOpts().Enabled().CreateIfMissing().VerifyChecksums()); - ASSERT_TRUE(err.ok()) << err.what(); - - checkExpectations(reindexer, ns, pattern, description); - - // remove storage - err = rt.reindexer->CloseNamespace(ns); - ASSERT_TRUE(err.ok()) << err.what(); - - reindexer::fs::RmDirAll(storagePath); - } - - void SpecialCheckForNull(const std::shared_ptr& reindexer, std::string_view ns, std::string_view firstItemPattern, - std::string_view itemPattern, std::string_view description, const std::string& storagePath) const { - specialCheckForNull(reindexer, ns, firstItemPattern, itemPattern, description); - validateReloadStateForNull(reindexer, ns, firstItemPattern, itemPattern, description, storagePath); - } - - static constexpr uint32_t IdStart = 2000; - -private: - static constexpr char kStoragePath[] = "/tmp/reindex/"; - static constexpr uint32_t itemNumber_ = 5; // NOTE: minimum 2 - - [[nodiscard]] std::string createNS(std::string_view ns, std::string_view itemPattern) { - std::string storage; - createNamespace(ns, storage); - generateItems(ns, itemPattern); - return storage; - } - - void createNamespace(std::string_view ns, std::string& storagePath) { - storagePath = kStoragePath; - storagePath.append(ns); - - auto err = rt.reindexer->EnableStorage(storagePath); - ASSERT_TRUE(err.ok()) << err.what(); - - err = rt.reindexer->OpenNamespace(ns, StorageOpts().Enabled().CreateIfMissing()); - EXPECT_TRUE(err.ok()) << err.what(); - - DefineNamespaceDataset(ns, {IndexDeclaration{"id", "hash", "int", IndexOpts().PK(), 0}}); - } - - void generateItems(std::string_view ns, std::string_view pattern) { - for (uint32_t idx = IdStart, sz = IdStart + itemNumber_; idx < sz; ++idx) { - Item item = NewItem(ns); - EXPECT_TRUE(item.Status().ok()) << item.Status().what(); - - const auto json = fmt::sprintf(pattern.data(), idx); - auto err = item.FromJSON(json); - ASSERT_TRUE(err.ok()) << err.what(); - - Upsert(ns, item); - } - } - - void checkIfItemJSONValid(QueryResults::Iterator& it, bool print = false) const { - reindexer::WrSerializer wrser; - Error err = it.GetJSON(wrser, false); - EXPECT_TRUE(err.ok()) << err.what(); - if (err.ok() && print) { - std::cout << wrser.Slice() << std::endl; - } - } - - void validateResults(const std::shared_ptr& reindexer, std::string_view ns, std::string_view pattern, - std::string_view field, const VariantArray& expectedValues, std::string_view description) const { - SCOPED_TRACE(description); - - QueryResults qr; - auto err = reindexer->Select("SELECT * FROM " + std::string(ns), qr); - EXPECT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(qr.Count(), itemNumber_); - - for (auto it : qr) { - Item item = it.GetItem(false); - checkIfItemJSONValid(it); - const auto json = item.GetJSON(); - ASSERT_NE(json.find(pattern), std::string::npos) << "JSON: " << json << ";\npattern: " << pattern; - const VariantArray values = item[field]; - ASSERT_EQ(values.size(), expectedValues.size()); - ASSERT_EQ(values.IsArrayValue(), expectedValues.IsArrayValue()); - for (size_t i = 0; i < values.size(); ++i) { - ASSERT_TRUE(values[i].Type().IsSame(expectedValues[i].Type())) - << values[i].Type().Name() << "!=" << expectedValues[i].Type().Name(); - if (values[i].Type().IsSame(reindexer::KeyValueType::Null())) { - continue; - } - - ASSERT_EQ(values[i], expectedValues[i]); - } - } - } - - void checkExpectations(const std::shared_ptr& reindexer, std::string_view ns, std::string_view pattern, - std::string_view description) const { - SCOPED_TRACE(description); - - QueryResults qr; - auto err = reindexer->Select("SELECT * FROM " + std::string(ns), qr); - EXPECT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(qr.Count(), itemNumber_); - - uint32_t idx = IdStart; - for (auto it : qr) { - Item item = it.GetItem(false); - checkIfItemJSONValid(it); - - const auto json = item.GetJSON(); - const auto extJson = fmt::sprintf(pattern.data(), idx); - ASSERT_EQ(json, extJson); - - ++idx; - } - } - - void getItems(const std::shared_ptr& reindexer, std::string_view ns, std::vector& items) const { - QueryResults qr; - auto err = reindexer->Select("SELECT * FROM " + std::string(ns), qr); - ASSERT_TRUE(err.ok()) << err.what(); - - items.clear(); - items.reserve(qr.Count()); - for (auto& it : qr) { - auto item = it.GetItem(false); - items.emplace_back(item.GetJSON()); - } - } - - void checkItems(const std::shared_ptr& reindexer, std::string_view ns, - const std::vector& items) const { - QueryResults qr; - auto err = reindexer->Select("SELECT * FROM " + std::string(ns), qr); - ASSERT_TRUE(err.ok()) << err.what(); - - ASSERT_EQ(items.size(), qr.Count()); - auto itItems = items.cbegin(); - auto itQR = qr.begin(); - auto endItems = items.cend(); - auto endQR = qr.end(); - for (; (itItems != endItems) && (itQR != endQR); ++itItems, ++itQR) { - auto item = itQR.GetItem(false); - ASSERT_EQ(*itItems, item.GetJSON()); - } - } - - void specialCheckForNull(const std::shared_ptr& reindexer, std::string_view ns, std::string_view firstItemPattern, - std::string_view itemPattern, std::string_view description) const { - SCOPED_TRACE(description); - - // first element should not update values, all others should be initialized to default values - // Note: but index array updates element type - QueryResults qr; - auto err = reindexer->Select("SELECT * FROM " + std::string(ns), qr); - EXPECT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(qr.Count(), itemNumber_); - - uint32_t idx = IdStart; - for (auto it : qr) { - Item item = it.GetItem(false); - checkIfItemJSONValid(it); - const auto json = item.GetJSON(); - const auto& pattern = (idx == IdStart) ? firstItemPattern : itemPattern; - const auto expJson = fmt::sprintf(pattern.data(), idx); - ASSERT_EQ(json, expJson); - - ++idx; - } - } - - void validateReloadStateForNull(const std::shared_ptr& reindexer, std::string_view ns, - std::string_view firstItemPattern, std::string_view itemPattern, std::string_view description, - const std::string& storagePath) const { - auto err = rt.reindexer->CloseNamespace(ns); - ASSERT_TRUE(err.ok()) << err.what(); - - err = rt.reindexer->OpenNamespace(ns, StorageOpts().Enabled().VerifyChecksums()); - ASSERT_TRUE(err.ok()) << err.what(); - - specialCheckForNull(reindexer, ns, firstItemPattern, itemPattern, description); - - // remove storage - err = rt.reindexer->CloseNamespace(ns); - ASSERT_TRUE(err.ok()) << err.what(); - - reindexer::fs::RmDirAll(storagePath); - } -}; - -TEST_F(IndexTupleTest, DISABLED_ScalarTest) { - static const std::string ns = "testNSScalar"; - const auto storage = CreateEmptyNamespace(ns); - - DoTestEmpty(rt.reindexer, ns, {"sparse", "text", "string", IndexOpts().Sparse()}, R"({"id":%d})", "add some sparse index. Do nothing"); - DoTestDefault(rt.reindexer, ns, {"text", "text", "string", IndexOpts()}, R"("text":"")", "text", {Variant("")}, - "add text scalar index. Add default value"); - DoTestEmpty(rt.reindexer, ns, {"text", "text", "string", IndexOpts()}, R"({"id":%d,"text":""})", "update text index. Do nothing"); - DoCallAndCheckError(rt.reindexer, ns, {"text", "hash", "int", IndexOpts()}, - "Index 'testNSScalar.text' already exists with different settings"); - DoTestDefault(rt.reindexer, ns, {"int", "hash", "int", IndexOpts()}, R"("int":0)", "int", {Variant(0)}, - "add int scalar index. Add default value"); - ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"text":"","int":0})", "reload ns (ScalarTest)", storage); -} - -TEST_F(IndexTupleTest, DISABLED_ScalarNestedTest) { - static const std::string ns = "testNSNested"; - const auto storage = CreateEmptyNamespace(ns); - - DoTestDefault(rt.reindexer, ns, {"obj.more.nested", {"obj.more.nested"}, "hash", "int64", IndexOpts()}, - R"("obj":{"more":{"nested":0}})", "obj.more.nested", {Variant(int64_t(0))}, "add new nested scalar index"); - DoTestEmpty(rt.reindexer, ns, {"obj.more.nested", {"obj.more.nested"}, "hash", "int64", IndexOpts()}, - R"({"id":%d,"obj":{"more":{"nested":0}}})", "update nested index. Do nothing"); - DoTestEmpty(rt.reindexer, ns, {"id+obj.more.nested", {"id", "obj.more.nested"}, "tree", "composite", IndexOpts{}}, - R"({"id":%d,"obj":{"more":{"nested":0}}})", "add new composite index. Do nothing"); - DoCallAndCheckError(rt.reindexer, ns, {"obj.more", {"obj.more"}, "hash", "string", IndexOpts()}, - "Invalid tag type value for KeyValueType: ''"); - DoCallAndCheckError(rt.reindexer, ns, {"obj", "hash", "int64", IndexOpts()}, "Invalid tag type value for KeyValueType: ''"); - DoTestDefault(rt.reindexer, ns, {"obj.near", {"obj.near"}, "tree", "string", IndexOpts()}, R"("obj":{"more":{"nested":0},"near":""})", - "obj.near", {Variant("")}, "add nested scalar index to root"); - DoTestDefault(rt.reindexer, ns, {"obj.nested.text", {"obj.nested.text"}, "hash", "string", IndexOpts()}, - R"("obj":{"more":{"nested":0},"near":"","nested":{"text":""}}})", "obj.nested.text", {Variant("")}, - "add nested another path scalar index"); - DoTestDefault(rt.reindexer, ns, {"obj.idx", {"obj.idx"}, "hash", "int64", IndexOpts()}, - R"("obj":{"more":{"nested":0},"near":"","nested":{"text":""},"idx":0})", "obj.idx", {Variant(int64_t(0))}, - "add nested 2nd level path scalar index"); - DoTestDefault(rt.reindexer, ns, {"obj.new.another.one", {"obj.new.another.one"}, "tree", "double", IndexOpts()}, - R"("obj":{"more":{"nested":0},"near":"","nested":{"text":""},"idx":0,"new":{"another":{"one":0.0}}}})", - "obj.new.another.one", {Variant(0.0)}, "add nested scalar index with multiple new path levels"); - DoCallAndCheckError(rt.reindexer, ns, {"boom", {"obj.new.another.one"}, "tree", "string", IndexOpts()}, - "Cannot add field with name 'boom' to namespace 'testNSNested'. Json path 'obj.new.another.one' already used" - " in field 'obj.new.another.one'"); - DoCallAndCheckError(rt.reindexer, ns, {"boom", {"obj.new.another.one.two"}, "hash", "int64", IndexOpts()}, - "Cannot add field with name 'boom' (jsonpath 'obj.new.another.one.two') and type 'int64' to namespace" - " 'testNSNested'. Already exists json path 'obj.new.another.one' with type 'double' in field" - " 'obj.new.another.one'. Rewriting is impossible"); - DoTestDefault( - rt.reindexer, ns, {"root2.more.nested", {"root2.more.nested"}, "hash", "int64", IndexOpts()}, - R"("obj":{"more":{"nested":0},"near":"","nested":{"text":""},"idx":0,"new":{"another":{"one":0.0}}},"root2":{"more":{"nested":0}})", - "root2.more.nested", {Variant(int64_t(0))}, "add new root with nested"); - DoTestDefault( - rt.reindexer, ns, {"boom", {"obj.new.another.one_ext"}, "hash", "int64", IndexOpts()}, - R"("obj":{"more":{"nested":0},"near":"","nested":{"text":""},"idx":0,"new":{"another":{"one":0.0,"one_ext":0}}},"root2":{"more":{"nested":0}})", - "obj.new.another.one_ext", {Variant(int64_t(0))}, "add new nested scalar index with name extension in last part"); - DoTestDefault( - rt.reindexer, ns, {"a-ha", {"a.ha"}, "hash", "int64", IndexOpts()}, - R"("obj":{"more":{"nested":0},"near":"","nested":{"text":""},"idx":0,"new":{"another":{"one":0.0,"one_ext":0}}},"root2":{"more":{"nested":0}},"a":{"ha":0})", - "a.ha", {Variant(int64_t(0))}, "add another nested scalar index on top level"); - ValidateReloadState( - rt.reindexer, ns, - R"({"id":%d,"obj":{"more":{"nested":0},"near":"","nested":{"text":""},"idx":0,"new":{"another":{"one":0.0,"one_ext":0}}},"root2":{"more":{"nested":0}},"a":{"ha":0}})", - "reload ns (ScalarNestedTest)", storage); -} - -TEST_F(IndexTupleTest, SparseItemTest) { - static const std::string ns = "testNSSparse"; - const auto storage = CreateSparseNamespace(ns); - - DoTestEmpty(rt.reindexer, ns, {"sparse1", {"fld1"}, "hash", "int", IndexOpts().Sparse()}, - R"({"id":%d,"fld1":1,"fld2":{"nested":"test"}})", "add some sparse index to present nested field. Do nothing"); - DoCallAndCheckError(rt.reindexer, ns, {"sparse2", {"fld2"}, "hash", "int", IndexOpts().Sparse()}, "Can't convert 'test' to number"); - DoCallAndCheckError(rt.reindexer, ns, {"sparse3", {"fld2.nested"}, "hash", "int", IndexOpts().Sparse()}, - "Can't convert 'test' to number"); - DoTestEmpty(rt.reindexer, ns, {"sparse2", {"fld2"}, "hash", "string", IndexOpts().Sparse()}, - R"({"id":%d,"fld1":1,"fld2":{"nested":"test"}})", "add some sparse index to present part path field. Do nothing"); - ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"fld1":1,"fld2":{"nested":"test"}})", "reload ns (SparseItemTest)", storage); -} - -TEST_F(IndexTupleTest, NestedUpdateTest) { - static const std::string ns = "testNSUpdate"; - const auto storage = CreateNamespace(ns); - - DoTestDefault( - rt.reindexer, ns, {"obj.nested", {"obj.nested"}, "hash", "string", IndexOpts()}, - R"("objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":"0"},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}])", - "obj.nested", VariantArray{Variant{"0"}}, "add obj.nested index - update field type"); - DoCallAndCheckError(rt.reindexer, ns, {"try_change_type", {"last.text"}, "hash", "int", IndexOpts()}, "Can't convert 'OK' to number"); - ValidateReloadState( - rt.reindexer, ns, - R"({"id":%d,"objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":"0"},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}]})", - "reload ns (NestedUpdateTest)", storage); -} - -// TODO: This test must be reenabled after #1353 -TEST_F(IndexTupleTest, DISABLED_ArrayTest) { - static const std::string ns = "testNSArray"; - const auto storage = CreateEmptyNamespace(ns); - - DoTestDefault(rt.reindexer, ns, {"array", "hash", "int", IndexOpts().Array()}, R"("array":[])", "array", {}, - "add int array index. Add empty array"); - DoTestDefault(rt.reindexer, ns, - {"obj.some.arr_1st", {"obj.some.array", "arr_fld", "obj.array"}, "hash", "int64", IndexOpts().Array(), 0}, - R"("array":[],"arr_fld":[])", "arr_fld", VariantArray{}.MarkArray(), "add array index. Add empty array"); - DoCallAndCheckError(rt.reindexer, ns, {"obj.some.array", {"obj.array"}, "hash", "int64", IndexOpts().Array(), 0}, - "Cannot add field with name 'obj.some.array' to namespace 'testNSArray'. Json path 'obj.array' already used in " - "field 'obj.some.arr_1st'"); - DoTestDefault(rt.reindexer, ns, - {"obj.some.new_array", {"obj.some.new_array", "arr_fld1", "arr_fld2"}, "hash", "int64", IndexOpts().Array(), 0}, - R"("array":[],"arr_fld":[],"arr_fld2":[])", "arr_fld2", VariantArray{}.MarkArray(), - "add another array index (chooses last of two). Add empty array"); - // TODO: This logic is disabled due to #1819 - DoTestDefault(rt.reindexer, ns, {"obj.new.array", {"obj.new.array"}, "hash", "int64", IndexOpts().Array(), 0}, - R"("array":[],"arr_fld":[],"arr_fld2":[]})" /*,"obj":{"new":{"array":[]}})"*/, "obj.new.array", VariantArray{}, - "add new nested (only) index. Add empty array"); - // TODO: This logic is disabled due to #1819 - DoTestDefault(rt.reindexer, ns, {"arr", "hash", "int64", IndexOpts().Array()}, - R"("array":[],"arr_fld":[],"arr_fld2":[],"arr":[]})" /*,"obj":{"new":{"array":[]}},"arr":[])"*/, "arr", VariantArray{}, - "add new field with nested (only) indexes. Add empty array"); - DoCallAndCheckError(rt.reindexer, ns, - {"arr_restriction", {"arr_fld3", "arr_fld4", "arr.some.arr_1st"}, "hash", "int64", IndexOpts().Array(), 0}, - "Cannot add field with name 'arr_restriction' (jsonpath 'arr.some.arr_1st') and type 'int64' to namespace" - " 'testNSArray'. Already exists json path 'arr' with type 'int64' in field 'arr'. Rewriting is impossible"); - DoTestEmpty(rt.reindexer, ns, {"new_sparse_array", {"new_sparse_array"}, "hash", "int64", IndexOpts().Array().Sparse(), 0}, - R"({"id":%d,"array":[],"arr_fld":[],"arr_fld2":[],"arr":[]})" /*,"obj":{"new":{"array":[]}},"arr":[]})"*/, - "add new sparse array index. Do nothing"); - ValidateReloadState(rt.reindexer, ns, - R"({"id":%d,"array":[],"arr_fld":[],"arr_fld2":[],"arr":[]})" /*,"obj":{"new":{"array":[]}},"arr":[]})"*/, - "reload ns (ArrayTest)", storage); -} - -// TODO: This test must be reenabled after #1353 -TEST_F(IndexTupleTest, DISABLED_ArrayNestedTest) { - static const std::string ns = "testNSArrayObj"; - const auto storage = CreateNamespace(ns); - - DoCallAndCheckError(rt.reindexer, ns, {"try_change_type", {"last.text"}, "hash", "int", IndexOpts().Array().Sparse()}, - "Can't convert 'OK' to number"); - DoCallAndCheckError(rt.reindexer, ns, {"try_change_type", {"last.text"}, "hash", "int", IndexOpts().Array()}, - "Can't convert 'OK' to number"); - // TODO: This logic is disabled due to #1819 - DoTestDefault( - rt.reindexer, ns, {"next.another.last", {"next.another.last"}, "hash", "string", IndexOpts().Array()}, - R"("objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}]})" /*,"next":{"another":{"last":[]}})"*/ - , - "next.another.last", VariantArray{}, "add nested index to field by new path. Add empty array"); - DoTestDefault( - rt.reindexer, ns, {"obj.alternative", {"obj.alternative"}, "hash", "string", IndexOpts().Array()}, - R"("objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0,"alternative":[]},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}]})" /*,"next":{"another":{"last":[]}})"*/ - , - "obj.alternative", VariantArray{}, "add nested index to field. Add empty array"); - - DoTestDefault( - rt.reindexer, ns, {"last.1st.2nd.ext", {"last.1st.2nd.ext"}, "hash", "string", IndexOpts().Array()}, - R"("objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0,"alternative":[]},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14,"ext":[]}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}]})" /*,"next":{"another":{"last":[]}})"*/ - , - "last.1st.2nd.ext.more", VariantArray{}, "add nested-nested index to field. Add empty array"); - DoCallAndCheckError(rt.reindexer, ns, {"last.1st.2nd.ext", {"last.alt", "last.1st.2nd.ext"}, "hash", "string", IndexOpts().Array()}, - "Index 'testNSArrayObj.last.1st.2nd.ext' already exists with different settings"); - // TODO: This logic is disabled due to #1819 - // DoTestDefault( - // rt.reindexer, ns, {"last.1st.2nd.ext.more", {"last.1st.2nd.ext.more"}, "hash", "string", IndexOpts().Array()}, - // R"("objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0,"alternative":[]},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14,"ext":{"more":[]}}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}],"next":{"another":{"last":[]}})", - // "last.1st.2nd.ext.more", VariantArray{}, "add nested-nested index to field. Add empty array"); - // DoCallAndCheckError(rt.reindexer, ns, - // {"last.1st.2nd.ext.more", {"last.alt", "last.1st.2nd.ext.more"}, "hash", "string", IndexOpts().Array()}, - // "Index 'testNSArrayObj.last.1st.2nd.ext.more' already exists with different settings"); - DoTestDefault( - rt.reindexer, ns, {"last.1st.ext", {"last.1st.ext"}, "hash", "string", IndexOpts().Array()}, - R"("objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0,"alternative":[]},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14,"ext":[]}},"ext":[]},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}]})" /*,"next":{"another":{"last":[]}})"*/ - , - "last.1st.ext", VariantArray{}, "add array index into the presented nested field. Add empty array"); - ValidateReloadState( - rt.reindexer, ns, - R"({"id":%d,"objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0,"alternative":[]},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14,"ext":[]}},"ext":[]},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}]})" /*,"next":{"another":{"last":[]}}})"*/ - , - "reload ns (ArrayNestedTest)", storage); -} - -TEST_F(IndexTupleTest, ArrayInToArrayTest) { - static const std::string ns = "testNSArrayArr"; - const auto storage = CreateNamespace(ns); - - // TODO: This logic is disabled due to #1819 - DoTestDefault( - rt.reindexer, ns, {"objs.more", {"objs.more"}, "hash", "string", IndexOpts().Array()}, - R"("objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":[3,2,1]},{"field":11},{"field":[9]}]}])", - "obj.more", VariantArray{}, "do not add anything into objects array"); - DoTestEmpty( - rt.reindexer, ns, {"arr.nested_arr.field", {"arr.nested_arr.field"}, "hash", "string", IndexOpts().Array()}, - R"({"id":%d,"objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":["3","2","1"]},{"field":"11"},{"field":["9"]}]}]})", - "add nested index to array array (update). Do nothing"); - DoTestEmpty( - rt.reindexer, ns, {"arr.new_fld", {"arr.new_fld"}, "hash", "string", IndexOpts().Array()}, - R"({"id":%d,"objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":["3","2","1"]},{"field":"11"},{"field":["9"]}]}]})", - "add nested index to array array. Do nothing"); - DoTestEmpty( - rt.reindexer, ns, {"arr.nested_arr.ext_fld", {"arr.nested_arr.ext_fld"}, "hash", "string", IndexOpts().Array()}, - R"({"id":%d,"objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":["3","2","1"]},{"field":"11"},{"field":["9"]}]}]})", - "add nested nested index to array array. Do nothing"); - ValidateReloadState( - rt.reindexer, ns, - R"({"id":%d,"objs":[{"fld":1},{"fld":2},{"fld":5}],"obj":{"nested":0},"last":{"text":"OK","1st":{"2nd":{"3rd":3.14}}},"arr":[{"nested_arr":[{"field":["3","2","1"]},{"field":"11"},{"field":["9"]}]}]})", - "reload ns (ArrayInToArrayTest)", storage); -} - -// TODO: This test must be reenabled after #1353 -TEST_F(IndexTupleTest, DISABLED_NestedOrderingTest) { - static const std::string ns = "testNSNestedOrdering"; - const auto storage = CreateEmptyNamespace(ns); - - DoTestDefault(rt.reindexer, ns, {"nest1", {"obj.more.nested"}, "hash", "int", IndexOpts()}, R"("obj":{"more":{"nested":0}})", "nest1", - VariantArray{Variant{0}}, "add nest1. Add default value"); - DoTestDefault(rt.reindexer, ns, {"nest2", {"obj.near"}, "hash", "int", IndexOpts()}, R"("obj":{"more":{"nested":0},"near":0})", "nest2", - VariantArray{Variant{0}}, "add nest2. Add default value"); - DoTestDefault(rt.reindexer, ns, {"nest3", {"obj.nestd.text"}, "text", "string", IndexOpts()}, - R"("obj":{"more":{"nested":0},"near":0,"nestd":{"text":""}})", "nest3", VariantArray{Variant{""}}, - "add nest3. Add default value"); - DoTestDefault(rt.reindexer, ns, {"nest11", {"obj.more.nested2"}, "hash", "int", IndexOpts()}, - R"("obj":{"more":{"nested":0,"nested2":0},"near":0,"nestd":{"text":""}})", "nest11", VariantArray{Variant{0}}, - "add nest11. Add default value"); - ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"obj":{"more":{"nested":0,"nested2":0},"near":0,"nestd":{"text":""}}})", - "reload ns (NestedDiffOrderingTest)", storage); -} - -// TODO: This test must be reenabled after #1353 -TEST_F(IndexTupleTest, DISABLED_NullTest) { - static const std::string ns = "testNSNull"; - const auto storage = CreateEmptyNamespace(ns); - - // update only one first item - { - const std::string sql = "UPDATE testNSNull SET fld1 = null, fld2 = [null, null] WHERE id = " + std::to_string(IdStart); - Query query = Query::FromSQL(sql); - QueryResults qr; - auto err = rt.reindexer->Update(query, qr); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(qr.Count(), 1); - } - - // add indexes (simple and array) - { - auto err = rt.reindexer->AddIndex(ns, {"fld1", "hash", "int", IndexOpts()}); - ASSERT_TRUE(err.ok()) << err.what(); - err = rt.reindexer->AddIndex(ns, {"fld2", "hash", "string", IndexOpts().Array()}); - ASSERT_TRUE(err.ok()) << err.what(); - } - - SpecialCheckForNull(rt.reindexer, ns, R"({"id":%d,"fld1":null,"fld2":["null","null"]})", R"({"id":%d,"fld1":0,"fld2":[]})", - "null values test", storage); -} - -// TODO: This test must be reenabled after #1353 -TEST_F(IndexTupleTest, DISABLED_FailTest) { - static const std::string ns = "testNSFail"; - const auto storage = CreateEmptyNamespace(ns); - - DoTestDefault(rt.reindexer, ns, {"nest", {"obj.nest"}, "hash", "int", IndexOpts()}, R"("obj":{"nest":0})", "nest", - VariantArray{Variant{0}}, "add nest. Add default value"); - DoTestDefault(rt.reindexer, ns, {"idx", {"idx"}, "-", "bool", IndexOpts()}, R"("obj":{"nest":0},"idx":false)", "idx", - VariantArray{Variant{false}}, "add idx. Add default value"); - ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"obj":{"nest":0},"idx":false})", "reload ns (FailTest)", storage); -} - -// TODO: This test must be reenabled after #1353 -TEST_F(IndexTupleTest, DISABLED_NestedArrayTest) { - static const std::string ns = "testNSNestedArray"; - const auto storage = CreateArrayNamespace(ns); - - // TODO: This logic is disabled due to #1819 - DoTestDefault(rt.reindexer, ns, {"obj.obj1.arr", {"obj.obj1.arr"}, "hash", "int", IndexOpts().Array()}, - R"("obj":{"val":10},"arr":[1,2,3])", "obj.obj1.arr", VariantArray{}, - // R"("obj":{"val":10,"obj1":{"arr":[]}},"arr":[1,2,3])", "obj.obj1.arr", VariantArray{}, - "add obj.obj1.arr. Add default value"); - DoTestDefault(rt.reindexer, ns, {"obj.arr", {"obj.arr"}, "hash", "int", IndexOpts().Array()}, - R"("obj":{"val":10,"arr":[]},"arr":[1,2,3])", "obj.arr", VariantArray{}, "add obj.arr. Add default value"); - ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"obj":{"val":10,"arr":[]},"arr":[1,2,3]})", "reload ns (NestedArrayTest)", storage); -} diff --git a/cpp_src/gtests/tests/unit/protobuf_test.cc b/cpp_src/gtests/tests/unit/protobuf_test.cc index 885f0c65f..a53958b1f 100644 --- a/cpp_src/gtests/tests/unit/protobuf_test.cc +++ b/cpp_src/gtests/tests/unit/protobuf_test.cc @@ -19,10 +19,10 @@ const std::string kStreetValue = "Miracle Street, "; const std::string kPostalCodeValue = "9745 123 "; const double kSalaryValue = 11238761238768.232342342; -TEST_F(ReindexerApi, ProtobufConvesrionTest) { +TEST_F(ReindexerApi, ProtobufConversionTest) { // Check protobuf for basic types (int/double/array) and double <-> int conversion // !!! This test is using schema from cpp_src/gtests/tests/proto/conversion.proto. - // !!! Protobuf indexes are not constant and depend from the internal reindexer::Schema implementation. + // !!! Protobuf indexes are not persistent and depend on the internal implementation of reindexer::Schema. // clang-format off const std::string schema = R"z( { @@ -93,7 +93,7 @@ TEST_F(ReindexerApi, ProtobufConvesrionTest) { TEST_F(ReindexerApi, ProtobufEasyArrayTest) { // Check protobuf for arrays and nested objects // !!! This test is using schema from cpp_src/gtests/tests/proto/easyarrays.proto. - // !!! Protobuf indexes are not constant and depend from the internal reindexer::Schema implementation. + // !!! Protobuf indexes are not persistent and depend on the internal implementation of reindexer::Schema. // clang-format off const std::string schema = R"z( { diff --git a/cpp_src/net/cproto/dispatcher.h b/cpp_src/net/cproto/dispatcher.h index bf9c69cf9..c8b0c6d2a 100644 --- a/cpp_src/net/cproto/dispatcher.h +++ b/cpp_src/net/cproto/dispatcher.h @@ -89,7 +89,7 @@ class Dispatcher { /// Set closer notifier /// @param object close class object - /// @param func function, to be called on connecion close + /// @param func function, to be called on connection close template void OnClose(K* object, void (K::*func)(Context& ctx, const Error& err)) { onClose_ = [=](Context& ctx, const Error& err) { (static_cast(object)->*func)(ctx, err); }; @@ -113,7 +113,7 @@ class Dispatcher { /// @return OnResponse callback reference const std::function& OnResponseRef() const noexcept { return onResponse_; } - /// Handle RPC fron the context + /// Handle RPC from the context /// @param ctx - RPC context Error Handle(Context& ctx) { if rx_likely (uint32_t(ctx.call->cmd) < uint32_t(handlers_.size())) { @@ -141,7 +141,7 @@ class Dispatcher { template ::value, int> = 0> static T get_arg(const Args& args, size_t index, const Context& ctx) { if (index >= args.size()) { - throw Error(errParams, "Invalid args of %s call; argument %d is not submited", CmdName(ctx.call->cmd), static_cast(index)); + throw Error(errParams, "Invalid args of %s call; argument %d is not submitted", CmdName(ctx.call->cmd), static_cast(index)); } return T(args[index]); } @@ -172,7 +172,7 @@ class Dispatcher { std::function logger_; std::function onClose_; - // This should be called from the connection thread only to prevet access to other connection's ClientData + // This should be called from the connection thread only to prevent access to other connection's ClientData std::function onResponse_; }; } // namespace cproto diff --git a/cpp_src/net/iserverconnection.h b/cpp_src/net/iserverconnection.h index b005a788c..76f0a7beb 100644 --- a/cpp_src/net/iserverconnection.h +++ b/cpp_src/net/iserverconnection.h @@ -32,7 +32,7 @@ class IServerConnection { /// Restart connection /// @param s - socket of the accepted connection. - /// @return true - if successfuly restarted, false - if connection can't be restarted. + /// @return true - if successfully restarted, false - if connection can't be restarted. virtual bool Restart(socket&& s) = 0; /// Attach connection to another listener loop. Must be called from thread of loop /// @param loop - another loop to bind diff --git a/cpp_src/server/grpc/reindexerservice.cc b/cpp_src/server/grpc/reindexerservice.cc index adae54294..496417876 100644 --- a/cpp_src/server/grpc/reindexerservice.cc +++ b/cpp_src/server/grpc/reindexerservice.cc @@ -516,8 +516,6 @@ Error ReindexerService::buildItems(WrSerializer& wrser, const reindexer::QueryRe break; } case EncodingType::PROTOBUF: { - ProtobufBuilder builder(&wrser, ObjType::TypeObject); - ProtobufBuilder array = builder.Array("items"); for (auto& it : qr) { status = it.GetProtobuf(wrser, false); if (!status.ok()) { diff --git a/cpp_src/server/httpserver.cc b/cpp_src/server/httpserver.cc index bd59f8a18..c0ca367ed 100644 --- a/cpp_src/server/httpserver.cc +++ b/cpp_src/server/httpserver.cc @@ -1501,17 +1501,12 @@ int HTTPServer::queryResultsProtobuf(http::Context& ctx, const reindexer::QueryR WrSerializer wrSer(ctx.writer->GetChunk()); ProtobufBuilder protobufBuilder(&wrSer); - int itemsField = kProtoQueryResultsFields.at(kParamItems); for (size_t i = offset; i < res.Count() && i < offset + limit; i++) { - auto item = protobufBuilder.Object(itemsField); auto it = res[i]; - auto i1 = item.Object(res.getNsNumber(it.GetItemRef().Nsid()) + 1); const auto err = it.GetProtobuf(wrSer, false); if (!err.ok()) { return ctx.Protobuf(err.code(), wrSer.DetachChunk()); } - i1.End(); - item.End(); } int aggregationField = kProtoQueryResultsFields.at(kParamAggregations); diff --git a/test/queries_test.go b/test/queries_test.go index 5c5ec2095..ee491e037 100644 --- a/test/queries_test.go +++ b/test/queries_test.go @@ -558,7 +558,7 @@ func TestQueries(t *testing.T) { panic(err) } - CheckTestItemsQueries(t, testCaseWithIDOnlyIndexe) + CheckTestItemsQueries(t, testCaseWithIDOnlyIndexes) }) t.Run("Sparse indexed queries", func(t *testing.T) { t.Parallel() @@ -1267,7 +1267,7 @@ var testCaseWithCommonIndexes = IndexesTestCase{ }, Item: TestItem{}, } -var testCaseWithIDOnlyIndexe = IndexesTestCase{ +var testCaseWithIDOnlyIndexes = IndexesTestCase{ Name: "TEST WITH ID ONLY INDEX", Namespace: "test_items_id_only", Options: sortDistinctOptions{